diff --git a/.codecov.yml b/.codecov.yml new file mode 100644 index 0000000..94b7dca --- /dev/null +++ b/.codecov.yml @@ -0,0 +1,21 @@ +# Codecov configuration for ilaflott/fremorizer +# Reference: https://docs.codecov.com/docs/codecovyml-reference + +coverage: + status: + project: + default: + target: auto + threshold: 1% + if_ci_failed: error + + patch: + default: + target: 60% # Minimum coverage for new code + threshold: 1% # Allow 1% decrease + if_ci_failed: error + +comment: + layout: "header, diff, flags, components, files, footer" + behavior: default + require_changes: false diff --git a/.cspell.json b/.cspell.json new file mode 100644 index 0000000..d120a3b --- /dev/null +++ b/.cspell.json @@ -0,0 +1,25 @@ +// configuration for spellcheck workflow +{ + "version": "0.2", + "language": "en", + "caseSensitive": false, + "dictionaryDefinitions": [ + { + "name": "ok-unknown-words", + "path": "./.cspell/ok-unknown-words.txt", + "addWords": false, + } + ], + "dictionaries": [ + "ok-unknown-words", + // "noaa-words", // TODO sort words into categories + // "gfdl-words", + // "software-words" + ], + "ignorePaths": [ + "cspell.json", + "*.cdl", + "build", + "fremorizer.egg-info" + ] +} diff --git a/.cspell/ok-unknown-words.txt b/.cspell/ok-unknown-words.txt new file mode 100644 index 0000000..691e280 --- /dev/null +++ b/.cspell/ok-unknown-words.txt @@ -0,0 +1,623 @@ +addlibs +aliq +allmatch +allvars +amip +AMIP +analysiscombined +analysisyaml +AOGCM +abstractproperty +apidoc +apptainer +arag +argparse +archivcl +areacello +argparser +asdict +atleast +averager +Averager +avgs +avgtype +avgr +avgvals +BADOPT +BADYAMLFILE +baremetal +Baremetal +basenames +Blanton +bldsetup +bnds +BNDS +bndvar +booleaness +BUILDROOT +bytedate +capfd +caplog +casedirs +catalogbuilder +cdotimavg +CCYY +CCYYMM +CCYYMMDD +cdlf +CDOTIMEAVERAGER +cfgs +cftime +chdir +CHDIR +checkoutscript +chunkstr +classmethod +clim +Clim +climatologies +Climatologies +climo +CLIMO +cloudrad +cmip +CMIP +cmor +Cmor +CMOR +CMORBITE +cmorization +cmorize +cmorizes +cmorizing +cmorized +cmory +cmoryaml +cobv +coef +combinedfile +combinedyaml +combinefail +compdir +compilecombined +compiledict +compilefile +compileinfo +compilelog +compileyaml +COMPOUT +compp +compval +contactslist +Containerzation +contextlib +contextmanager +conv +CPLD +cppdefs +CPPDEFS +createscript +cshrc +cstring +cubedsphere +cylc +Cylc +CYLC +datavars +daterange +datestring +DCLIMATE +debuglog +decsiontree +defaultxy +delz +denit +DHAVE +difmxybo +dimname +DINTERNAL +dirstrings +distclean +divc +DMAXFIELDMETHODS +DMAXFIELDS +docstrings +Dorkey +dset +DSPMD +dunp +dura +ecpe +emis +Emon +encodekey +encodeval +enot +ensmem +errormsg +ESGF +evap +excinfo +execinfo +execrunscript +exitstatus +experimentname +expname +EBronx +EXPNAME +expyaml +Falses +fcadet +fcased +fdet +ffetot +filenotfound +flithdet +fnfeso +fnoxic +fntot +fout +fptot +freanalysis +freapp +frecatalog +frecheck +frecheckexample +frecmor +fremor +fremorizer +fregrid +frelist +fremake +Fremake +frenc +frenctols +frenctools +frepp +Frepp +frepythontools +frepytools +FREPYTOOLSTIMEAVERAGER +frerun +frerunexample +fretarget +frevars +freyaml +freyamltools +fsitot +genbadge +genfunctions +geolat +geolon +Geophys +geotherm +getenot +getmakeline +getncattr +gettarget +GETTID +gfdlfremake +giccm +globus +Globus +gpfs +gridding +gridfile +gridfiles +gridlocation +gridspec +gridsy +gridtiles +gridyaml +gtas +helpdesk +hgrid +histval +hpcme +hpcmini +hpoint +htotal +iamnotreallyused +ierr +IFMS +Imom +indexlist +indir +INDIR +infolog +inlinevar +infile +infiles +inputdir +inputfile +INPUTFILE +inputfiles +intakebuilder +intercomparison +intercomparisons +interp +Interp +INTERP +intrafile +ISMIP +isodatetime +isort +ivar +ixin +jaon +jhan +jhanbigmem +Jinjafilter +jsondecode +jsondecodeerr +jsonschema +keyerr +Kiihne +Koder +Laflotte +landuse +lastindex +latlon +latxlon +lazygroup +LDFLAGS +levelname +levhalf +lhdf +libstring +Libstring +libyaml +linkerflags +linkline +linklinecreate +linklineon +listtools +Lmod +Lmon +lnetcdf +lons +lwdn +lwflx +lwup +mainyaml +makefilefre +matchlist +mdtf +MDTF +meke +mekep +mentees +mergetime +meshgrid +metaclass +metavars +metomi +microphys +miniforge +minmax +mkmf +Mkmf +mkmfclone +mktemplate +mocsy +modelyaml +modindex +modis +modname +modulefiles +MODULESHOME +monniker +mosaicfile +moveaxis +mpicc +msvs +muldpm +multifile +multijob +MULTIJOB +myavger +mymodule +mypackage +mypy +namenopath +nbhome +ncatted +ncattr +ncattrs +ncatts +nccheck +nccmp +ncdf +ncdump +ncfile +ncgen +NCGEN +ncks +ncontact +ncrc +netcdfs +nctool +nctool's +nctools +nctools's +ndarray +ndims +Nenv +Nikonov +Nikonov's +nlat +NLAT +nlon +NLON +nncattr +noaa +NOAA +noarch +noleap +noqa +nosec +notransfer +notused +nparallel +ntiles +ntimes +nullyaml +numpy +nvariables +nxyp +oceangrid +Oday +Odec +omip +Omon +openmp +OPENMP +optparse +origdir +orog +outf +outfiledir +outpath +outputdir +OUTPUTDIR +outputfile +OUTPUTFILE +overgeneral +parallelizations +parseable +parseyaml +pathlib +pathnames +pcheck +PCMDI +pdclim +pfull +phalf +pids +PLACEHOLD +platformfre +platforminfo +platformsdict +platformsfile +platformsyaml +platformyaml +platformyamlcontent +plev +plevel +posteriori +postprocesses +ppan +PPAN +ppcombined +ppcomps +ppcompstle +ppcompstyle +ppip +ppsettingsy +ppsettingsyaml +ppst +ppval +ppyaml +ppyamls +preanalysis +prec +precip +prereq +psfile +pstring +ptest +ptmp +pygtk +pylint +pylintrc +pytest +Pytest +pytests +pythonic +pytools +pyyaml +qpoint +quietlog +rainwat +Rbite +rcfile +Rcommander +RDHPCS +redissolution +reengineered +reff +refinediag +regrid +Regrid +REGRID +regriddable +regridded +regridding +Regridding +regrids +regrions +relaces +renku +reqstring +retstat +rgxs +rgxy +Rization +Rize +Rized +Rizing +Rmon +rootdir +ROOTDIR +rtype +runscript +samp +secondstage +Sergey +setncattr +setncatts +settingsyaml +setuptools +shflx +shortvars +SIGPIPE +smth +snowwat +SOURCEDIR +spack +spackloads +SPHINXBUILD +SPHINXOPTS +sphum +splitmon +srces +SRCROOT +srcs +stddevs +stdoutput +Stofferahn +strat +subdaily +subdirs +subhr +subp +subtool +Subtool +subtools +Subtools +surfresp +swdn +swup +tarfiles +targ +targetfre +tasmax +tasmin +TAVG +testingtestingtestingtesting +thefiles +timavg +timavgcsh +timeaveraged +timeaverager +timeavgs +timelevels +timesteps +Timesteps +timmean +timsum +tmpdirpath +tmpdirs +tmpfs +TOAR +toctree +topdir +tripolar +ttest +truncatedate +truncatedateformat +turb +twostep +typealias +typevar +ucomp +Umip +underbaked +unlimdim +unmsk +untar +untarred +unwgt +Utheri +valerr +validatefail +variablescan +vardict +varlist +varnames +varsfre +varsize +vcomp +vcoords +vect +Wagura +wgts +Whitlock +withism +xarray +xcomp +xcomponents +xdaily +Xlnd +Xocean +Xocn +xstr +Xtest +xtmp +xvalues +xyinterp +XYINTERP +yamlcomp +yamlconfig +yamldict +YAMLDIR +yamler +yamlfile +YAMLFILE +yamlfilepath +YAMLFILES +yamlfre +yamlpath +YAMLPATH +yamls +Yamls +yamltools +yamlvalidate +ydict +ymlsources +ymonmean +ymonstddev +yseasmean +yseasstddev +yvalues +YYYY +YYYYMM +YYYYMMDD +YYYYMMDDHH +YYYYMMDDHH:mm +zaxis +zbounds +zerovars +zfactor +zonavg +zsurf +zxvf +donwell +hartfield +longbourn +pemberley +openedfile +lonxlat +njobs +RDHPCS +wnps diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 0000000..c365469 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,20 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: '' +assignees: '' + +--- + +**Describe the bug** + + +**To Reproduce** + + +**Expected behavior** + + +**Additional context** +Add any other context about the problem here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 0000000..bbcbbe7 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: '' +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/PULL_REQUEST_TEMPLATE/release_procedure.md b/.github/PULL_REQUEST_TEMPLATE/release_procedure.md new file mode 100644 index 0000000..4dc8fcd --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE/release_procedure.md @@ -0,0 +1,41 @@ +## Release Versioning Procedure + +## Checklist +### fremorizer changes +* [ ] 1. Verify that git submodules in fremorizer reflect the latest state (or certain commit/tag) of the upstream repositories. + + - If not, consult the manager of the upstream repository and determine whether the update should be included in this release. + - If so, ask the sub-project maintainer to tag the upstream repository + + Open a PR to commit the submodule updates in `fremorizer`, solicit a review, and merge the PR. + + - **Submodules**: + - `fremorizer/tests/test_files/cmip6-cmor-tables` + - `fremorizer/tests/test_files/cmip7-cmor-tables` + + **Note**: The release schedules of these submodules may vary from that of fremorizer + +* [ ] 2. Create a tag in the fremorizer repository (testing tag or release tag) + + Locally this can be done with: + + ``` + git tag -a + git push origin + ``` + + - For the *testing tags*, follow the structure: `[year].[major].[minor].[testing tag]` + + - `[year].[major].[minor].alpha[iteration]`: alpha tags include major code breaking changes + - `[year].[major].[minor].beta[iteration]`: beta tags include releases candidates for testing + + - For the *full release tag*, follow the structure: `[year].[major].[minor]` + + After the tag is pushed, CI will trigger the creation of a PR changing any reference to the previous tag with the new tag. + Verify the tagged release is present [here](https://github.com/ilaflott/fremorizer/releases>) + +* [ ] 3. For a full release (only), create the github release associated with the correct tag and generate the release notes. + + - In the release notes, be sure to link any alpha and beta tags that were tested for the release + +* [ ] 4. Navigate to [noaa-gfdl conda channel](https://anaconda.org/NOAA-GFDL/fremorizer) and verify that the last upload date corresponds to the date of this release and that the release number is correct. diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..14b64b4 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,16 @@ +## Describe your changes + +## Issue ticket number and link (if applicable) +Fixes #XXX (replace XXX with the issue number and GitHub will autolink the PR to the issue) +## Checklist before requesting a review + +- [ ] I ran my code +- [ ] I tried to make my code readable +- [ ] I tried to comment my code +- [ ] I wrote a new test, if applicable +- [ ] I wrote new instructions/documentation, if applicable +- [ ] I ran pytest and inspected it's output +- [ ] I ran pylint and attempted to implement some of it's feedback +- [ ] No print statements; all user-facing info uses logging module + +*Note: If you are a code maintainer updating the tag or releasing a new fremorizer version, please use the `release_procedure.md` template. To quickly use this template, open a new pull request, choose your branch, and add `?template=release_procedure.md` to the end of the url.* diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml new file mode 100644 index 0000000..1743d1a --- /dev/null +++ b/.github/workflows/build_conda.yml @@ -0,0 +1,63 @@ +name: build_conda +on: + pull_request: + branches: + +# cancel running jobs if theres a newer push +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + condabuild: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: + - name: Checkout Files + uses: actions/checkout@v4 + + - name: Setup Conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-activate-base: true + miniforge-version: latest + channels: conda-forge,noaa-gfdl + + - name: Configure Conda + run: | + echo "removing main and r channels from defaults" + conda config --remove channels defaults || true + conda config --remove channels main || true + conda config --remove channels r || true + + echo "setting strict channel priority" + conda config --set channel_priority strict + + echo "setting anaconda_upload to no" + conda config --set anaconda_upload no + + echo "printing conda config just in case" + conda config --show + + - name: Conda install conda-build + run: | + echo "conda install conda-build" + conda install conda-forge::conda-build + + - name: Build fremorizer Conda Package + run: | + echo "conda building fremorizer package and outputting as a tarball" + mkdir -p /tmp/fremorizer-tarball + conda build --package-format tar.bz2 --output-folder /tmp/fremorizer-tarball . + + - name: Upload fremorizer Tarball + uses: actions/upload-artifact@v4 + with: + name: fremorizer-tarball + path: /tmp/fremorizer-tarball/noarch/fremorizer-*.tar.bz2 + if-no-files-found: error diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml new file mode 100644 index 0000000..993d40d --- /dev/null +++ b/.github/workflows/create_test_conda_env.yml @@ -0,0 +1,63 @@ +name: create_test_conda_env +on: + pull_request: + branches: + +# cancel running jobs if theres a newer push +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + conda-test: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: + - name: Checkout Files + uses: actions/checkout@v4 + + - name: Setup Conda + uses: conda-incubator/setup-miniconda@v3 + with: + activate-environment: fremorizer + environment-file: environment.yaml + auto-activate-base: false + miniforge-version: latest + + - name: Configure Conda + run: | + echo "removing main and r channels from defaults" + conda config --remove channels defaults || true + conda config --remove channels main || true + conda config --remove channels r || true + + echo "setting strict channel priority" + conda config --set channel_priority strict + + echo "printing conda config just in case" + conda config --show + + - name: Install fremorizer + run: | + pip install . + + - name: Run pytest + run: | + pytest --cov=fremorizer --cov-config=coveragerc fremorizer/tests/ -v + + - name: Upload Coverage to Codecov + if: ${{ always() }} + uses: codecov/codecov-action@v5 + with: + token: ${{ secrets.CODECOV_TOKEN }} + verbose: true + + - name: Run pylint + if: ${{ always() }} + run: | + pylint --rcfile pylintrc fremorizer/ diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml new file mode 100644 index 0000000..8334952 --- /dev/null +++ b/.github/workflows/publish_conda.yml @@ -0,0 +1,48 @@ +name: publish_conda +on: + push: + branches: + - main + tags: + - '*' + +permissions: + contents: read + +jobs: + condapublish: + runs-on: ubuntu-latest + defaults: + run: + shell: bash -l {0} + steps: + - name: Checkout Files + uses: actions/checkout@v4 + + - name: Setup Conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-activate-base: true + channels: conda-forge,noaa-gfdl + + - name: Configure Conda + run: | + echo "setting strict channel priority" + conda config --set channel_priority strict + + echo "setting anaconda_upload to yes" + conda config --set anaconda_upload yes + + echo "printing conda config just in case" + conda config --show + + - name: Conda install conda-build + run: | + echo "conda install conda-build" + conda install conda-forge::conda-build conda-forge::anaconda-client + + - name: Build fremorizer Conda Package + run: | + echo "conda building fremorizer package" + export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} + conda build . diff --git a/.github/workflows/spell_check.yml b/.github/workflows/spell_check.yml new file mode 100644 index 0000000..3a9c5f2 --- /dev/null +++ b/.github/workflows/spell_check.yml @@ -0,0 +1,84 @@ +name: 'Check spelling' +on: + pull_request: + branches: + - main + push: + branches: + - main + +permissions: + contents: read + +jobs: + spellcheck: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: streetsidesoftware/cspell-action@v7 + continue-on-error: true + with: + # Define glob patterns to filter the files to be checked. + # Use a new line between patterns to define multiple + # patterns. The default is to check ALL files that were + # changed in in the pull_request or push. + # Note: `ignorePaths` defined in cspell.json still apply. + # Example: + # files: | + # **/*.{ts,js} + # !dist/**/*.{ts,js} + # + # Default: ALL files + files: '' + + # Check files and directories starting with `.`. + # - "true" - glob searches will match against `.dot` files. + # - "false" - `.dot` files will NOT be checked. + # - "explicit" - glob patterns can match explicit `.dot` + # patterns. + check_dot_files: explicit + + # The point in the directory tree to start spell checking. + # Default: . + root: '.' + + # Notification level to use with inline reporting of + # spelling errors. + # Allowed values are: warning, error, none + # Default: warning + inline: warning + + # Reports flagged / forbidden words as errors. + # If true, errors will still be reported even if `inline` + # is "none" + treat_flagged_words_as_errors: false + + # Generate Spelling suggestions. + suggestions: false + + # Determines if the action should be failed if any spelling + # issues are found. Allowed values are: true, false + # Default: true + strict: true + + # Limit the files checked to the ones in the pull request or + # push. + # This should be changed to true later. + incremental_files_only: false + + # Path to `cspell.json` + config: './.cspell.json' + + # Log progress and other information during the action + # execution. + # Default: false + verbose: false + + # Use the `files` setting found in the CSpell configuration + # instead of `input.files`. + use_cspell_files: false + + # Set how unknown words are reported. + # Allowed values are: all, simple, typos, flagged + # Default: all + report: all diff --git a/.github/workflows/update_tag.yaml b/.github/workflows/update_tag.yaml new file mode 100644 index 0000000..92be9f4 --- /dev/null +++ b/.github/workflows/update_tag.yaml @@ -0,0 +1,57 @@ +name: Update Tag Version References + +on: + push: + tags: + - '*' + +permissions: + contents: read + pull-requests: write + +jobs: + tag-update: + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Fetch tags + run: | + git fetch --tags origin + + - name: Get the new tag + id: get_tag + run: | + echo "tag=${GITHUB_REF#refs/tags/}" >> $GITHUB_OUTPUT + + - name: Get previous tag + id: get_prev_tag + run: | + new_tag="${{ steps.get_tag.outputs.tag }}" + prev_tag=$(git tag --sort=-version:refname | grep -v "^${new_tag}$" | head -n 1) + echo "prev_tag=${prev_tag}" >> $GITHUB_OUTPUT + + - name: Replace old tag with new tag in all relevant files + run: | + new_tag="${{ steps.get_tag.outputs.tag }}" + old_tag="${{ steps.get_prev_tag.outputs.prev_tag }}" + # Only replace if previous tag exists + if [ -n "$old_tag" ]; then + # all .md files + find . -type f -name "*.md" -exec sed -i "s/$old_tag/$new_tag/g" {} + + # all .py files + find . -type f -name "*.py" -exec sed -i "s/$old_tag/$new_tag/g" {} + + # docs (all files) + find docs -type f -exec sed -i "s/$old_tag/$new_tag/g" {} + + fi + + - name: Create pull request + uses: peter-evans/create-pull-request@v7.0.6 + with: + base: main + branch: update-version-post-release + branch-suffix: timestamp + delete-branch: true + title: Update refs to version number post-release + body: automated change, replaces references to the old version number with the new version number upon releases. diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..b8fe9ea --- /dev/null +++ b/.gitignore @@ -0,0 +1,160 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Vim files (not including *~) +*.swp +*.swo + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +share/python-wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.nox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +*.py,cover +.hypothesis/ +.pytest_cache/ +cover/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 +db.sqlite3-journal + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +.pybuilder/ +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# IPython +profile_default/ +ipython_config.py + +# pyenv +# .python-version + +# pipenv +#Pipfile.lock + +# poetry +#poetry.lock + +# pdm +.pdm.toml + +# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm +__pypackages__/ + +# Celery stuff +celerybeat-schedule +celerybeat.pid + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ +.dmypy.json +dmypy.json + +# Pyre type checker +.pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ + +# other +*~ +*\#* + +# PyCharm +.idea/ + +.vscode/* +.vscode + +# files pulled/created via testing +*.nc +tmp/ + +# autogen'd by doc build +docs/fremorizer.* +docs/modules.rst diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 0000000..d4c20ce --- /dev/null +++ b/.gitmodules @@ -0,0 +1,6 @@ +[submodule "fremorizer/tests/test_files/cmip6-cmor-tables"] + path = fremorizer/tests/test_files/cmip6-cmor-tables + url = https://github.com/pcmdi/cmip6-cmor-tables.git +[submodule "fremorizer/tests/test_files/cmip7-cmor-tables"] + path = fremorizer/tests/test_files/cmip7-cmor-tables + url = https://github.com/WCRP-CMIP/cmip7-cmor-tables.git diff --git a/.readthedocs.yaml b/.readthedocs.yaml new file mode 100644 index 0000000..9e053d5 --- /dev/null +++ b/.readthedocs.yaml @@ -0,0 +1,26 @@ +# Read the Docs configuration file +# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details +# Required +version: 2 + +# Build documentation using lightweight Python environment +# Heavy dependencies (netCDF4, cmor, xarray, etc.) are mocked in docs/conf.py +# This significantly speeds up builds and reduces complexity +build: + os: ubuntu-24.04 + tools: + python: "3.11" + jobs: + pre_install: + # Install minimal dependencies needed for doc build + - pip install sphinx renku-sphinx-theme sphinx-rtd-theme click pyyaml + pre_build: + # Generate API documentation from source + - sphinx-apidoc --ext-autodoc --output-dir docs fremorizer/ --separate + +# Configure Sphinx +sphinx: + configuration: docs/conf.py + + + diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..db4c10a --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,29 @@ +# fremorizer Code of Conduct + +## Purpose +Modeling Systems Division, and GFDL as whole, seeks to create a friendly, safe and welcoming environment for all, regardless of gender, sexual orientation, ability, ethnicity, socioeconomic status, and religion. In order to protect this collaborative environment, the following Code of Conduct sets out a series of guidelines to follow while interacting with the fremorizer repository, and its contributors. This code applies equally to mentors, mentees, and independent contributors. + +## Expected Behavior +* Demonstrating empathy and kindness toward other people +* Being respectful of differing opinions, viewpoints, and experiences +* Giving and gracefully accepting constructive feedback +* Accepting responsibility and apologizing to those affected by our mistakes, + and learning from the experience +* Focusing on what is best not just for us as individuals, but for the overall + community + +## Unacceptable Behavior +* The use of sexualized language or imagery, and sexual attention or advances of + any kind +* Trolling, insulting or derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or email address, + without their explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Addressing Grievances: +Unacceptable behavior from any community member will not be tolerated. Anyone asked to stop unacceptable behavior is expected to comply immediately. Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by emailing gfdl.climate.model.info@noaa.gov + +## Attribution +This Code of Conduct is adapted from the [Contributor Covenant](https://www.contributor-covenant.org/version/1/4/code-of-conduct.html), version 1.4, the [Citizen Code of Conduct](https://web.archive.org/web/20200330154000/http://citizencodeofconduct.org/), and the [GFDL Code of Conduct](https://www.gfdl.noaa.gov/wp-content/uploads/2022/05/GFDL_Code_of_Conduct.pdf). diff --git a/CODE_STYLE.md b/CODE_STYLE.md new file mode 100644 index 0000000..eaa9cf9 --- /dev/null +++ b/CODE_STYLE.md @@ -0,0 +1,104 @@ +# FRE Code Style Guide + +Disclaimer: This style guide is still in development. + +Follow these Style Guidelines when contributing to the `fremorizer` repository. + +## Code Checklist + +Checklist before requesting a review: + +- [ ] I ran my code +- [ ] I tried to make my code readable +- [ ] I tried to comment my code +- [ ] I wrote a new test, if applicable +- [ ] I wrote new instructions/documentation, if applicable +- [ ] I ran pytest and inspected it's output +- [ ] I ran pylint and attempted to implement some of it's feedback +- [ ] No print statements; all user-facing info uses logging module + +## General + +- Trim all trailing whitespace from every line +- Lines must be <= 120 characters long (including comments) + +## Documentation Tips + +Here are some things to keep in mind when writing documentation: + - Keep sentences short + - Avoid the use of pronouns such as this, that, and it + - Use the imperative mood for procedures + - E.g. Instead of "You should install Conda" say “Install Conda” + - Write in active voice rather than passive + - Write objectively (minimize humor, jargon, idioms, etc) + +Useful Resources: + - https://docs.openstack.org/doc-contrib-guide/writing-style/general-writing-guidelines.html + +## Inline Python Documentation Requirements + +Document classes and functions with docstrings. These docstrings should contain field lists documenting arguments and +returns. Field lists are sequences of fields marked up: `:fieldname: Field content`. You do not need to document +click interface functions with field lists. + +Document classes, variables and functions with type hinting. It is encouraged to document click interface functions +with type hinting. Type hinting is when you annotate functions and variables with the expected Python type. +Annotations are described in [PEP 3107](https://peps.python.org/pep-3107/). + +Useful Resources + - https://docs.python.org/3/library/typing.html + - https://mypy.readthedocs.io/en/stable/cheat_sheet_py3.html (Note: fremorizer is not dependent on mypy) + - https://www.sphinx-doc.org/en/master/usage/restructuredtext/basics.html#field-lists + - https://sphinx-rtd-tutorial.readthedocs.io/en/latest/docstrings.html + +This example shows how to properly document a class and two functions with docstrings containing field lists and type +hinting: + +```python +from typing import Any, List, ClassVar + +class MyClass(object): + """ + Description for class + + :cvar str var3: description + :ivar str var1: description, initial value: par1 + :ivar str var2: description, initial value: par2 + """ + + var3: ClassVar[str] = "I am a class variable" + + def __init__(self, par1: int, par2: int): + self.var1 = par1 # instance variables + self.var2 = par2 + +def func_with_return_and_optional_param(a: int, c: List[int] = [1,2]) -> Any: + """ + summary + + :param a: description + :type a: int + :param c: description, defaults to [1,2] + :type c: list, optional + :raises ValueError: description + :return: description + :rtype: type + + .. note:: This is a note + .. warning:: This is a warning + """ + + if a > 10: + raise ValueError("a is more than 10") + return c + +def simple_func(foo: str): + """ + summary + + :param foo: description + :type foo: str + """ + + pass +``` diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..e701eff --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,67 @@ +## **For Developers** + +* Developers are free to use this repository's `README.md` to familiarize with the CLI and save time from having to install any dependencies, but development within a Conda environment is heavily recommended regardless +* Gain access to the repository with `git clone --recursive git@github.com:ilaflott/fremorizer.git` or your fork's link (recommended) and an SSH RSA key + - Once inside the repository, developers can test local changes by running a `pip install .` inside of the root directory to install the fremorizer package locally with the newest local changes on top of the installed Conda fremorizer dependencies + - Optional: load fre-nctools into your PATH to gain access to regridding and certain time-averaging routines (e.g., `module load fre-nctools` on GFDL/Gaea systems) + - Test as a normal user would use the CLI +* Create a GitHub issue to reflect your contribution's background and reference it with Git commits + +### **Opening Pull Requests and Issues** +Please use one of the templates present in this repository to open a PR or an issue, and fill out the template to the best of your ability. + +### **Adding New Commands/Tools - Checklist** + +If there is *no* subdirectory created for the new tool command group you are trying to develop, there are a few steps to follow: + + 1. Create a subdirectory for the tool group inside the /fre folder; i.e. `/fre/tool` + 2. Add an `__init__.py` inside of the new subdirectory + - this will contain as many lines as needed for each tool subcommand feature (function/class), following the syntax: `from .subCommandScript import subCommandFunction` or `from .subCommandScript import subCommandClass` + - at the end of the `__init__.py` file, add an `__all__` [variable](https://realpython.com/python-all-attribute/), following [this syntax](https://github.com/ilaflott/fremorizer/blob/refactoring/fre/pp/__init__.py): `__all__ = ["subCommandFunction1", "subCommandFunction2", "subCommandClass1", "subCommandClass2"]` + - the purpose of these lines are to enable `fre.py` to invoke them using its own [`__init__.py`](https://github.com/ilaflott/fremorizer/blob/refactoring/fre/__init__.py + 3. Create separate files to house the code implementation for each different subcommand; *do not* include any Click decorators for your function, except for `@click.command` + - Define the function normally with its usual arguments; the Click decorators to prompt them will instead go into `fre[tool].py` + 4. Remember to import any needed packages/dependencies in your subcommand script file + 5. _As of second refactoring_: Create a file named `fre[tool].py` (i.e `fremake.py`) + 6. In `fre[tool].py`, import all script commands from the tool's `__init__.py` file (i.e. `from .tool import subCommandFunction1, subCommandClass1, etc.`), and create a `@click.group` called `[tool]Cli` that is simply passed to the `if __name__ == "__main__":` block at the bottom of the file + 7. Using `@[tool]Cli.command()`, add the `@click.option` and any other [Click attributes/decorators](https://click.palletsprojects.com/en/8.1.x/api/#click.command) needed + - The commands within `fre[tool].py` must contain an additional decorator after the arguments, options, and other command components: `@click.pass_context` + - Add `context` and the other decorator attributes into the function declaration (i.e. `def subCommand(context, yaml, platform, target)`) + - Add a `""" - [description] """` to help describe the command and pass `context.forward(subCommandFunction)` inside of the command to let it invoke the functions from outside files + 8. If the tool group is not already added into the `__init__.py` in the /fre folder, add it using `from .tool import *` + 9. With the lazy groups implemented in `lazy_group.py`, all that needs to be done is to add to the `lazy_subcommands` defined inside of the main `@click.group`, `fre`, inside of `fre.py` + - Add the line: `"[tool]": ".[tool].fre[tool].[tool]Cli"` + - (Recommended): If the update is significant, consider incrementing the version number within [`setup.py`](https://github.com/ilaflott/fremorizer/blob/088ad363392b3bf187119d8970c22779d59aaed0/setup.py#L5) to reflect and signify the changes + 10. Test by running `pip install .` from the root level of the directory, and running `fre`, followed by any subcommands necessary + +### **Adding Tools From Other Repositories** + +* Currently, the solution to this task is to approach it using Conda packages. The tool that is being added must reside within a repository that contains a meta.yaml that includes Conda dependencies like the one in this repository and ideally a setup.py (may be subject to change due to deprecation) that may include any potentially needed pip dependencies + - Once published as a Conda package, ideally on the [NOAA-GFDL channel](https://anaconda.org/NOAA-GFDL), an addition can be made to the "run" section under the "requirements" category in the meta.yaml of the fremorizer following the syntax `channel::package` + - On pushes to the main branch, the [package](https://anaconda.org/ilaflott/fremorizer) will automatically be updated using the workflow file + +### **MANIFEST.in** + +* In the case where non-python files like templates, examples, and outputs are to be included in the fremorizer package, MANIFEST.in can provide the solution. Ensure that the file exists within the correct folder, and add a line to the MANIFEST.in following [this syntax](https://setuptools.pypa.io/en/latest/userguide/miscellaneous.html) + - For more efficiency, if there are multiple files of the same type needed, the MANIFEST.in addition can be something like `recursive-include fre/fre(subTool) *.fileExtension` which would recursively include every file matching that fileExtension within the specified directory and its respective subdirectories. Currently, fremorizer recursively includes every python and non-python file inside of /fre, although this may change in the future + - `setup.py` handles these files using [setuptools and namespace package finding](https://setuptools.pypa.io/en/latest/userguide/package_discovery.html) + +### **Example /fre Directory Structure** +``` +. +├── __init__.py +├── fre.py +├── README-tool-template.md +├── lazy_group.py +├── /[tool] +│   ├── __init__.py +│   ├── fre[tool].py +│   ├── README.md +│   ├── [subCommandScript].py +│   └── /[optional-submodule] +``` + +## **Helpful Links** +* [Official Click Documentation](https://click.palletsprojects.com/en/8.1.x/api/) +* [`setup.py` Key Words](https://setuptools.pypa.io/en/latest/references/keywords.html) +* [`meta.yaml` Documentation](https://docs.conda.io/projects/conda-build/en/latest/resources/define-metadata.html) diff --git a/LICENSE.md b/LICENSE.md new file mode 100644 index 0000000..2bb9ad2 --- /dev/null +++ b/LICENSE.md @@ -0,0 +1,176 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS \ No newline at end of file diff --git a/MANIFEST.in b/MANIFEST.in new file mode 100644 index 0000000..efc6c03 --- /dev/null +++ b/MANIFEST.in @@ -0,0 +1,3 @@ +include README.md +include LICENSE.md +recursive-include fremorizer * diff --git a/PHASE_2.5_EQUIVALENCE_TESTING.md b/PHASE_2.5_EQUIVALENCE_TESTING.md new file mode 100644 index 0000000..b553d2e --- /dev/null +++ b/PHASE_2.5_EQUIVALENCE_TESTING.md @@ -0,0 +1,300 @@ +# Phase 2.5: Equivalence Testing with fre-cli + +This document outlines the plan for validating that fremorizer produces equivalent output to fre.cmor in fre-cli. + +## Objective + +Establish confidence that fremorizer operates equivalently to the fre.cmor submodule it was derived from, ensuring no functional regressions were introduced during the package separation. + +## Approach + +### 1. Test Data Preparation + +**Setup Requirements:** +- Install both fremorizer and fre-cli in separate environments +- Prepare identical test datasets for both tools +- Use existing test fixtures from `fremorizer/tests/test_files/` + +**Test Cases:** +- Use the same input netCDF files +- Use the same CMOR table configurations (from cmor-tables submodule) +- Use the same experiment configuration JSON files +- Use the same variable lists + +### 2. Comparative Testing Strategy + +**A. CLI Command Equivalence** + +Test that equivalent CLI commands produce identical results: + +```bash +# fre-cli command +fre cmor run -d -l -r -p -o + +# fremorizer command +fremor run -d -l -r
-p -o +``` + +**B. Output Comparison** + +For each test case, compare: +1. **File structure**: Verify output directory structure matches +2. **File metadata**: Compare global attributes, variable attributes +3. **Data values**: Verify numerical equivalence (within tolerance for floating-point precision) +4. **File naming**: Confirm output files follow same naming conventions + +**C. Test Scenarios** + +Minimum test coverage should include: + +1. **Basic CMORization** (`fremor run`) + - Single variable, monthly data + - Multiple variables, different frequencies + - Different MIP tables (Omon, Amon, etc.) + +2. **YAML-based workflow** (`fremor yaml`) + - Simple YAML configuration + - Multi-component workflow + - Start/stop year filtering + +3. **Variable discovery** (`fremor find`) + - Search across multiple tables + - Variable presence validation + +4. **Config generation** (`fremor config`) + - Automatic YAML generation from directory tree + - Variable list creation + +5. **Edge cases** + - Grid label handling (gn, gr) + - Calendar type variations + - Time chunk boundaries + +### 3. Implementation Plan + +**Option A: Automated CI Workflow** (Recommended) + +Create `.github/workflows/equivalence_test.yml`: + +```yaml +name: equivalence_test + +on: + pull_request: + branches: + - main + workflow_dispatch: + +jobs: + compare-with-fre-cli: + runs-on: ubuntu-latest + steps: + - name: Checkout fremorizer + uses: actions/checkout@v4 + path: fremorizer + + - name: Checkout fre-cli + uses: actions/checkout@v4 + with: + repository: noaa-gfdl/fre-cli + path: fre-cli + + - name: Setup Conda with miniforge + uses: conda-incubator/setup-miniconda@v3 + with: + miniforge-version: latest + auto-activate-base: false + + - name: Remove unwanted conda channels + run: | + conda config --remove channels defaults || true + conda config --remove channels main || true + conda config --remove channels r || true + conda config --set channel_priority strict + + - name: Create fremorizer environment + run: | + conda create -n fremorizer-test python>=3.11 -c conda-forge -c noaa-gfdl + conda activate fremorizer-test + cd fremorizer + pip install . + + - name: Create fre-cli environment + run: | + conda create -n fre-cli-test python>=3.11 -c conda-forge -c noaa-gfdl + conda activate fre-cli-test + cd fre-cli + pip install . + + - name: Run comparison tests + run: | + # Execute comparison script (to be created) + python fremorizer/tests/equivalence/compare_with_fre_cli.py + + - name: Upload comparison report + if: always() + uses: actions/upload-artifact@v4 + with: + name: equivalence-report + path: equivalence_report.md +``` + +**Option B: Manual Testing Script** + +Create `fremorizer/tests/equivalence/compare_with_fre_cli.py`: +- Automated comparison script that can be run locally or in CI +- Generates detailed report of differences (if any) +- Returns exit code 0 for equivalence, non-zero for differences + +### 4. Comparison Script Structure + +```python +# fremorizer/tests/equivalence/compare_with_fre_cli.py + +import subprocess +import tempfile +import netCDF4 as nc +import numpy as np +from pathlib import Path + +class EquivalenceTest: + def __init__(self, test_name): + self.test_name = test_name + self.fremorizer_env = "fremorizer-test" + self.fre_cli_env = "fre-cli-test" + + def run_fremorizer(self, cmd_args): + """Run fremor command in fremorizer environment""" + # Implementation + + def run_fre_cli(self, cmd_args): + """Run fre cmor command in fre-cli environment""" + # Implementation + + def compare_outputs(self, dir1, dir2): + """Compare output directories""" + # Compare file structure + # Compare netCDF files + # Compare metadata + # Return detailed comparison results + + def compare_netcdf_files(self, file1, file2): + """Deep comparison of two netCDF files""" + # Compare dimensions + # Compare variables + # Compare attributes + # Compare data values (with tolerance) + +def main(): + tests = [ + ("basic_monthly_amon", {...}), + ("yaml_workflow", {...}), + ("multi_variable", {...}), + # More test cases + ] + + results = [] + for test_name, test_config in tests: + test = EquivalenceTest(test_name) + result = test.run_and_compare(test_config) + results.append(result) + + # Generate report + generate_report(results) + + # Exit with appropriate code + if all(r.passed for r in results): + sys.exit(0) + else: + sys.exit(1) +``` + +### 5. Success Criteria + +Equivalence is established when: + +1. ✅ All test cases produce matching output file structures +2. ✅ All netCDF files have identical dimensions and variables +3. ✅ All metadata (global and variable attributes) match +4. ✅ All data values are numerically equivalent (within 1e-10 relative tolerance for floating-point) +5. ✅ No differences in file naming conventions +6. ✅ Same behavior for error cases (e.g., missing variables, invalid configs) + +### 6. Known Acceptable Differences + +Document any intentional differences (if any exist): + +- **Import paths**: `from fre.cmor.X` → `from fremorizer.X` (internal only) +- **CLI command**: `fre cmor` → `fremor` (user-facing difference is intentional) +- **Package metadata**: Version strings, package names in attributes (acceptable) +- **Logging format**: Minor differences in log output format (acceptable if semantic content matches) + +### 7. Implementation Timeline + +**Prerequisites:** +- ✅ Phase 1-2 complete (package foundation and source files) +- ✅ CI/CD pipelines working (miniforge, correct channels) +- ⏳ Current CI workflows passing + +**Implementation Steps:** + +1. **Create comparison script** (1-2 days) + - Basic netCDF comparison logic + - Output structure validation + - Report generation + +2. **Define test cases** (1 day) + - Select representative test scenarios + - Prepare test data + - Document expected behavior + +3. **Create CI workflow** (1 day) + - Set up dual environment installation + - Integrate comparison script + - Configure artifact uploads + +4. **Execute and validate** (2-3 days) + - Run tests locally first + - Debug any differences found + - Document acceptable vs. problematic differences + - Iterate until equivalence achieved + +5. **Documentation** (1 day) + - Update this document with results + - Add equivalence badge to README + - Document any known differences + +**Total estimated effort**: 6-8 days + +### 8. Maintenance + +After initial equivalence is established: + +- Run equivalence tests on every PR that modifies core functionality +- Update test cases when new features are added +- Re-validate equivalence before each release +- Before implementing Phase 3 features, ensure equivalence baseline is solid + +### 9. Notes and Considerations + +**Dependencies:** +- Requires `netCDF4` Python library (already in environment.yaml) +- May need `xarray` for easier netCDF comparisons +- Consider using `numpy.testing.assert_allclose` for numerical comparisons + +**Challenges:** +- Timestamps in output may differ (creation time, processing time) - these should be excluded from comparison +- Random number generation (if any) may cause differences - need deterministic test data +- File system ordering may affect output order - comparison should be order-independent + +**Alternative Approaches:** +- Use existing test suite with both tools and compare pytest output +- Focus on subset of most critical functionality first +- Consider comparing outputs on real-world data from GFDL workflows + +## References + +- fre-cli repository: https://github.com/noaa-gfdl/fre-cli +- fre.cmor submodule: `fre/cmor/` in fre-cli +- fremorizer test suite: `fremorizer/tests/` +- NetCDF comparison tools: `netCDF4`, `xarray`, `nccmp` (external) diff --git a/PHASE_3_DEFERRED_FEATURES.md b/PHASE_3_DEFERRED_FEATURES.md new file mode 100644 index 0000000..7e297f9 --- /dev/null +++ b/PHASE_3_DEFERRED_FEATURES.md @@ -0,0 +1,107 @@ +# Phase 3: Deferred Features from fre.cmor + +This document tracks features from open fre.cmor pull requests that are intentionally deferred for future implementation in fremorizer. Each item corresponds to a specific PR in the [fre-cli repository](https://github.com/noaa-gfdl/fre-cli). + +## Purpose + +Phase 3 features represent enhancements and improvements that were in progress in the fre.cmor submodule. These are deferred to focus on: +1. Establishing a stable, working independent fremorizer package (Phase 1-2) +2. Validating equivalence with existing fre-cli functionality (Phase 2.5) +3. Ensuring CI/CD pipelines are robust and reliable + +Once the foundation is solid and equivalence testing is complete, these features can be incorporated systematically. + +## Phase 3 Items + +### 1. PR #826: Replace nccmp with netCDF4 in tests +- **Source**: https://github.com/noaa-gfdl/fre-cli/pull/826 +- **Description**: Replace external `nccmp` tool dependency with Python `netCDF4` library for comparing netCDF files in tests +- **Impact**: Would eliminate external tool dependency, making tests more portable and reliable +- **Priority**: High - currently 9 tests fail due to missing nccmp tool + +### 2. PR #832: Harden branded-variable disambiguations +- **Source**: https://github.com/noaa-gfdl/fre-cli/pull/832 +- **Description**: Improve handling of branded variable name disambiguation (e.g., when multiple variables map to same target) +- **Impact**: More robust variable handling in edge cases +- **Priority**: Medium + +### 3. PR #833: Improved omission tracking +- **Source**: https://github.com/noaa-gfdl/fre-cli/pull/833 +- **Description**: Better tracking and reporting of variables that are omitted during CMORization +- **Impact**: Improved user feedback and debugging capabilities +- **Priority**: Medium + +### 4. PR #834: New `fremor init` command for config fetching +- **Source**: https://github.com/noaa-gfdl/fre-cli/pull/834 +- **Description**: Add new CLI subcommand to fetch/initialize CMOR configuration files +- **Impact**: Improved user experience for initial setup +- **Priority**: Medium - new feature, not a fix + +### 5. PR #836: Informative error on mip_era/table format mismatch +- **Source**: https://github.com/noaa-gfdl/fre-cli/pull/836 +- **Description**: Provide clear error messages when MIP era and table format don't match +- **Impact**: Better user experience and faster debugging +- **Priority**: Medium + +### 6. PR #837: Accept CF calendar aliases (noleap/365_day, etc.) +- **Source**: https://github.com/noaa-gfdl/fre-cli/pull/837 +- **Description**: Support CF convention calendar aliases (e.g., "noleap" as alias for "365_day") +- **Impact**: Improved compatibility with various input data conventions +- **Priority**: Medium + +### 7. PR #838: CMIP7 flavored tests +- **Source**: https://github.com/noaa-gfdl/fre-cli/pull/838 +- **Description**: Add test cases specifically for CMIP7 conventions and requirements +- **Impact**: Ensure CMIP7 compatibility and catch CMIP7-specific issues +- **Priority**: High - CMIP7 support is important for future-proofing + +### 8. PR #846: Variable list semantics (map modeler vars to MIP table names) +- **Source**: https://github.com/noaa-gfdl/fre-cli/pull/846 +- **Description**: Improve variable list handling to better map model variable names to MIP table variable names +- **Impact**: More flexible and intuitive variable mapping +- **Priority**: Medium + +### 9. PR #817: Update cmor to 3.14.2 +- **Source**: https://github.com/noaa-gfdl/fre-cli/pull/817 +- **Description**: Update the CMOR library dependency from current version to 3.14.2 +- **Impact**: Access to latest CMOR features and bug fixes +- **Priority**: Medium - should verify compatibility before upgrading + +## Implementation Strategy + +When ready to implement Phase 3 features: + +1. **Create individual GitHub issues** for each item above with: + - Link to the original fre-cli PR + - Description of the feature/fix + - Any fremorizer-specific considerations + - Testing requirements + +2. **Prioritize based on**: + - User impact (e.g., PR #826 fixes current test failures) + - Dependencies between features + - Alignment with CMIP6/CMIP7 timelines + +3. **Implementation approach**: + - Review the original PR in fre-cli for implementation details + - Adapt code to fremorizer's independent package structure + - Ensure all changes maintain test coverage + - Update documentation as needed + - Validate no regressions in existing functionality + +4. **Testing requirements**: + - All existing tests must continue to pass + - New tests should be added for new features + - Phase 2.5 equivalence tests should still pass after each feature addition + +## Notes + +- Phase 2.5 (equivalence testing with fre-cli) should be completed **before** implementing Phase 3 features +- Each Phase 3 feature should be implemented in a separate PR for easier review and rollback if needed +- Consider creating a project board to track Phase 3 implementation progress + +## References + +- Original fre-cli repository: https://github.com/noaa-gfdl/fre-cli +- fre.cmor submodule path: `fre/cmor/` in fre-cli +- This fremorizer PR: https://github.com/ilaflott/fremorizer/pull/1 diff --git a/README.md b/README.md index dc5d1db..89430fb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,105 @@ # fremorizer -model output rewriter for FRE/FMS based models + +Model output rewriter (CMORizer) for FRE/FMS based models. + +`fremorizer` is an independent package extracted from the `fre.cmor` submodule of +[fre-cli](https://github.com/NOAA-GFDL/fre-cli). It rewrites climate model output +files with CMIP-compliant metadata for downstream publishing using the +[CMOR](https://cmor.llnl.gov/) library. + +## Installation + +### Via pip +```bash +pip install fremorizer +``` + +### Via conda +```bash +conda install -c noaa-gfdl -c conda-forge fremorizer +``` + +### From source +```bash +git clone https://github.com/ilaflott/fremorizer.git +cd fremorizer +pip install . +``` + +## Usage + +The CLI entry point is `fremor`. It maps directly from the `fre cmor` subcommand: + +``` +# fre-cli equivalent: fre -vv -l logfile.txt cmor run [OPTIONS] +# fremorizer equivalent: fremor -vv -l logfile.txt run [OPTIONS] +``` + +### Subcommands + +```bash +fremor run # Rewrite climate model output files with CMIP-compliant metadata +fremor yaml # Process a CMOR YAML configuration file +fremor find # Find variables in MIP tables +fremor varlist # Create a simple variable list from netCDF files +fremor config # Generate a CMOR YAML configuration from a pp directory tree +``` + +### Verbosity and Logging + +```bash +fremor -v run ... # INFO level logging +fremor -vv run ... # DEBUG level logging +fremor -q run ... # ERROR level only (quiet) +fremor -l log.txt run ... # Log to file (appends) +``` + +### Example: CMORize ocean data + +```bash +fremor run \ + -d /path/to/input/netcdf/dir \ + -l /path/to/varlist.json \ + -r /path/to/CMIP6_Omon.json \ + -p /path/to/exp_config.json \ + -o /path/to/output/dir +``` + +## Requirements + +- Python >= 3.11 +- cftime +- click +- cmor +- netCDF4 +- numpy +- pyyaml + +## Development + +```bash +# Create conda environment +conda env create -f environment.yaml +conda activate fremorizer + +# Install in editable mode +pip install -e . + +# Run tests +pytest fremorizer/tests/ -v + +# Run linter +pylint --rcfile pylintrc fremorizer/ +``` + +## Relationship to fre-cli + +`fremorizer` is a near-exact copy of the `fre.cmor` submodule from +[NOAA-GFDL/fre-cli](https://github.com/NOAA-GFDL/fre-cli), extracted as an +independent package. The `fremor yaml` subcommand optionally depends on +`fre-cli`'s `yamltools` module for YAML consolidation. + +## License + +Apache License 2.0 — see [LICENSE.md](LICENSE.md) + diff --git a/coveragerc b/coveragerc new file mode 100644 index 0000000..cb43dc1 --- /dev/null +++ b/coveragerc @@ -0,0 +1,3 @@ +[run] +omit = + fremorizer/tests/* diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..ed88099 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = . +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/api.rst b/docs/api.rst new file mode 100644 index 0000000..c07c375 --- /dev/null +++ b/docs/api.rst @@ -0,0 +1,2 @@ +.. automodule:: fremorizer + :members: diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 0000000..9d7bd5e --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,41 @@ +# Configuration file for the Sphinx documentation builder. +# +# For the full list of built-in configuration values, see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +import sys +import datetime as dt +from pathlib import Path + +sys.path.insert(0, str(Path('..').resolve())) + +from fremorizer import __version__ as pkg_version + +# -- Project information ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = 'fremorizer' +copyright = f'{dt.datetime.now().year}, NOAA-GFDL MSD Workflow Team' +author = 'NOAA-GFDL MSD Workflow Team' +release = pkg_version # type: ignore + +# -- General configuration --------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration +extensions = ['sphinx.ext.autodoc'] +exclude_patterns = [] + +# Mock imports for dependencies not needed during doc build +# This allows Sphinx to build docs without installing heavy dependencies +autodoc_mock_imports = [ + 'cftime', + 'cmor', + 'netCDF4', + 'numpy', + 'xarray', + 'pandas', + 'pytest', +] + +# -- Options for HTML output ------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output +html_theme = 'renku' diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 0000000..de06c41 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,17 @@ +.. fremorizer documentation master file + +========================================== +Welcome to ``fremorizer``'s documentation! +========================================== + +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + api + +Indices +======= + +* :ref:`modindex` +* :ref:`search` diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..03125f4 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,3 @@ +sphinx +renku-sphinx-theme +sphinx-rtd-theme diff --git a/environment.yaml b/environment.yaml new file mode 100644 index 0000000..6c413b7 --- /dev/null +++ b/environment.yaml @@ -0,0 +1,15 @@ +name: fremorizer +channels: + - conda-forge + - noaa-gfdl +dependencies: + - python >=3.11 + - conda-forge::cftime + - conda-forge::click >=8.2 + - conda-forge::cmor >=3.14 + - conda-forge::netcdf4 >=1.7 + - conda-forge::numpy >=2 + - conda-forge::pyyaml + - conda-forge::pytest + - conda-forge::pytest-cov + - conda-forge::pylint diff --git a/fremorizer/__init__.py b/fremorizer/__init__.py new file mode 100644 index 0000000..ebe2a28 --- /dev/null +++ b/fremorizer/__init__.py @@ -0,0 +1,21 @@ +""" +module init file for fremorizer. sets the version attribute, and sets up a fre_logger +""" + +import logging +import os +version = os.getenv("GIT_DESCRIBE_TAG", "2026.01.alpha1") +__version__ = version + +fre_logger = logging.getLogger(__name__) + +FORMAT = "[%(levelname)5s:%(filename)24s:%(funcName)24s] %(message)s" +logging.basicConfig(level = logging.WARNING, + format = FORMAT, + filename = None, + encoding = 'utf-8' ) + +from .cmor_mixer import cmor_run_subtool +from .cmor_finder import cmor_find_subtool, make_simple_varlist +from .cmor_yamler import cmor_yaml_subtool +from .cmor_config import cmor_config_subtool diff --git a/fremorizer/cmor_config.py b/fremorizer/cmor_config.py new file mode 100644 index 0000000..e49fb0b --- /dev/null +++ b/fremorizer/cmor_config.py @@ -0,0 +1,229 @@ +""" +CMOR YAML Configuration Generator +================================= + +This module powers the ``fre cmor config`` command, generating a CMOR YAML configuration +file that ``fre cmor yaml`` can consume. It scans a post-processing directory tree for +available components and time-series data, cross-references found variables against MIP +tables, and produces the structured YAML needed for CMORization. + +Functions +--------- +- ``cmor_config_subtool(...)`` + +.. note:: This module was derived from quick_script.py prototyping work. +""" + +import glob +import logging +from pathlib import Path + +from .cmor_finder import make_simple_varlist +from .cmor_constants import EXCLUDED_TABLE_SUFFIXES + +fre_logger = logging.getLogger(__name__) + + +def _filter_mip_tables(mip_tables_dir: str, mip_era: str): + """ + Glob MIP table JSON files from the given directory, filtering out + non-variable-entry tables (grids, coordinates, etc.). + + :param mip_tables_dir: Path to directory containing MIP table JSON files. + :type mip_tables_dir: str + :param mip_era: MIP era string, e.g. 'cmip6' or 'cmip7'. + :type mip_era: str + :return: List of paths to MIP table JSON files. + :rtype: list[str] + """ + era_upper = mip_era.upper() + all_tables = glob.glob(f'{mip_tables_dir}/{era_upper}_*.json') + + filtered = [] + for table_path in all_tables: + table_stem = Path(table_path).stem # e.g. "CMIP7_ocean" + suffix = table_stem.split('_', maxsplit=1)[1] if '_' in table_stem else '' + if suffix not in EXCLUDED_TABLE_SUFFIXES: + filtered.append(table_path) + + fre_logger.debug('filtered MIP tables (%d of %d): %s', + len(filtered), len(all_tables), filtered) + return filtered + + +def cmor_config_subtool( + pp_dir: str, + mip_tables_dir: str, + mip_era: str, + exp_config: str, + output_yaml: str, + output_dir: str, + varlist_dir: str, + freq: str = 'monthly', + chunk: str = '5yr', + grid: str = 'g99', + overwrite: bool = False, + calendar_type: str = 'noleap' +): + """ + Generate a CMOR YAML configuration file from a post-processing directory tree. + + Scans ``pp_dir`` for pp-component directories, cross-references found variables + against MIP tables, writes per-component variable lists, and emits a structured + YAML that ``fre cmor yaml`` can later consume. + + :param pp_dir: Root post-processing directory containing per-component subdirectories. + :type pp_dir: str + :param mip_tables_dir: Directory containing MIP table JSON files. + :type mip_tables_dir: str + :param mip_era: MIP era identifier, e.g. 'cmip6' or 'cmip7'. + :type mip_era: str + :param exp_config: Path to JSON experiment/input configuration file expected by CMOR. + :type exp_config: str + :param output_yaml: Path to write the output CMOR YAML configuration. + :type output_yaml: str + :param output_dir: Root output directory for CMORized data. + :type output_dir: str + :param varlist_dir: Directory in which per-component variable list JSON files are written. + :type varlist_dir: str + :param freq: Temporal frequency string, e.g. 'monthly', 'daily'. Default 'monthly'. + :type freq: str + :param chunk: Time chunk string, e.g. '5yr', '10yr'. Default '5yr'. + :type chunk: str + :param grid: Grid label anchor name, e.g. 'g99', 'gn'. Default 'g99'. + :type grid: str + :param overwrite: If True, overwrite existing variable list files. Default False. + :type overwrite: bool + :param calendar_type: Calendar type string, e.g. 'noleap', '360_day'. Default 'noleap'. + :type calendar_type: str + :raises FileNotFoundError: If pp_dir or mip_tables_dir do not exist. + :raises ValueError: If no MIP tables are found after filtering. + :return: Path to the written output YAML file. + :rtype: str + """ + # ---- validate inputs ---- + if not Path(pp_dir).is_dir(): + raise FileNotFoundError(f'pp_dir does not exist: {pp_dir}') + if not Path(mip_tables_dir).is_dir(): + raise FileNotFoundError(f'mip_tables_dir does not exist: {mip_tables_dir}') + if not Path(exp_config).is_file(): + raise FileNotFoundError(f'exp_config does not exist: {exp_config}') + + # ensure output directories exist + Path(varlist_dir).mkdir(parents=True, exist_ok=True) + Path(output_yaml).parent.mkdir(parents=True, exist_ok=True) + + # ---- gather MIP tables ---- + mip_tables = _filter_mip_tables(mip_tables_dir, mip_era) + if not mip_tables: + raise ValueError( + f'no MIP tables found in {mip_tables_dir} for era {mip_era} after filtering') + + # ---- discover pp components ---- + ppcompdirs = sorted(glob.glob(f'{pp_dir}/*')) + fre_logger.info('found %d entries in pp_dir', len(ppcompdirs)) + + # ---- build YAML lines ---- + lines = [ + '', + 'cmor:', + ' start:', + ' *CMOR_START', + ' stop:', + ' *CMOR_STOP', + ' calendar_type:', + f" '{calendar_type}'", + ' mip_era:', + f" '{mip_era}'", + ' exp_json:', + f" '{exp_config}'", + ' directories:', + ' pp_dir: &pp_dir', + f" '{pp_dir}'", + ' table_dir: &table_dir', + f" '{mip_tables_dir}'", + ' outdir:', + f" '{output_dir}'", + ' table_targets:', + ] + + era_upper = mip_era.upper() + + for mip_table in sorted(mip_tables): + table_name = Path(mip_table).stem.split('.')[0].split('_')[1] # e.g. CMIP7_ocean + fre_logger.debug('processing mip_table = %s', table_name) + + appended_table_header = False + + for entry in ppcompdirs: + component_name = Path(entry).name + + variable_list = f'{varlist_dir}/{era_upper}_{table_name}_{component_name}.list' + #variable_list = f'{varlist_dir}/{table_name}_{component_name}.list' + + # optionally regenerate + if Path(variable_list).exists() and overwrite: + fre_logger.debug('varlist %s exists, unlinking to recreate because overwrite=True', + Path(variable_list).name) + Path(variable_list).unlink() + + if not Path(entry).is_dir(): + fre_logger.debug('entry %s is not a directory, skipping', entry) + continue + + # check for time-series data + data_series_present = [ + Path(ds).name for ds in glob.glob(f'{entry}/*') + if Path(ds).is_dir() + ] + if 'ts' not in data_series_present: + fre_logger.debug('no ts directory in %s, skipping', entry) + continue + + dir_targ = f'{entry}/ts/{freq}/{chunk}' + if not Path(dir_targ).is_dir(): + fre_logger.debug('target dir %s does not exist, skipping', dir_targ) + continue + + if len(glob.glob(f'{dir_targ}/*nc')) < 1: + fre_logger.debug('no nc files in %s, skipping', dir_targ) + continue + + try: + make_simple_varlist( + dir_targ=dir_targ, + output_variable_list=variable_list, + json_mip_table=mip_table + ) + except Exception: + fre_logger.warning( + 'variable list creation failed for %s %s %s', + dir_targ, variable_list, mip_table + ) + continue + + if Path(variable_list).exists(): + if not appended_table_header: + lines.append('') + lines.append(f" - table_name: '{table_name}'") + lines.append(f" freq: '{freq}'") + lines.append( ' gridding:') + lines.append(f' <<: *{grid}') + lines.append( ' target_components:') + appended_table_header = True + + lines.append(f" - component_name: '{component_name}'") + lines.append(f" variable_list: '{variable_list}'") + lines.append(" data_series_type: 'ts'") + lines.append(" chunk: *PP_CMIP_CHUNK") + + + # ---- write output YAML ---- + if Path(output_yaml).exists(): + Path(output_yaml).unlink() + + with open(output_yaml, 'w', encoding='utf-8') as out: + out.write('\n'.join(lines)) + + fre_logger.info('wrote CMOR YAML configuration to %s', output_yaml) + return output_yaml diff --git a/fremorizer/cmor_constants.py b/fremorizer/cmor_constants.py new file mode 100644 index 0000000..887eb95 --- /dev/null +++ b/fremorizer/cmor_constants.py @@ -0,0 +1,100 @@ +""" +fre.cmor constants +================== + +Centralized constants for the ``fre cmor`` subpackage. Every hard-coded +value that was previously scattered across ``cmor_mixer``, ``cmor_helpers``, +``cmor_config``, ``cmor_finder``, and ``cmor_yamler`` now lives here so that +each module imports from a single, transparent location. + +Sections +-------- +- **Vertical-coordinate classification** – lists that partition the accepted + vertical dimension names into physical categories. +- **CMOR module defaults** – arguments passed to ``cmor.setup()``. +- **CMIP7 brand disambiguation** – mapping from input netCDF vertical + dimension names to MIP-table dimension names. +- **Archive / filesystem paths** – locations of gold-standard data sets. +- **MIP-table filtering** – suffixes used to exclude non-variable-entry + tables when scanning a MIP-tables directory. +- **Output / display flags** – behavioural toggles for CLI and finder output. +""" + +import cmor + + +# --------------------------------------------------------------------------- +# Vertical-coordinate classification (used by cmor_mixer) +# --------------------------------------------------------------------------- +ACCEPTED_VERT_DIMS = [ + "z_l", "landuse", + "plev39", "plev30", "plev19", "plev8", + "height2m", + "level", "lev", "levhalf", +] + +NON_HYBRID_SIGMA_COORDS = [ + "landuse", + "plev39", "plev30", "plev19", "plev8", + "height2m", +] + +ALT_HYBRID_SIGMA_COORDS = ["level", "lev", "levhalf"] + +DEPTH_COORDS = ["z_l"] + + +# --------------------------------------------------------------------------- +# CMOR module defaults (passed to cmor.setup in cmor_mixer) +# --------------------------------------------------------------------------- +CMOR_NC_FILE_ACTION = cmor.CMOR_REPLACE +CMOR_VERBOSITY = cmor.CMOR_NORMAL +CMOR_EXIT_CTL = cmor.CMOR_NORMAL +CMOR_MK_SUBDIRS = 1 +CMOR_LOG = None + + +# --------------------------------------------------------------------------- +# CMIP7 brand disambiguation (used by cmor_helpers.filter_brands) +# --------------------------------------------------------------------------- +# Maps input netCDF vertical dimension names to their CMIP7 MIP-table +# equivalents. Dimensions whose names already match (e.g. plev39, height2m) +# need no entry; the look-up falls back to using the input name directly. +INPUT_TO_MIP_VERT_DIM = { + "z_l": "olevel", + "level": "alevel", + "lev": "alevel", + "levhalf": "alevhalf", +} + + +# --------------------------------------------------------------------------- +# Archive / filesystem paths (used by cmor_helpers) +# --------------------------------------------------------------------------- +ARCHIVE_GOLD_DATA_DIR = '/archive/gold/datasets' +CMIP7_GOLD_OCEAN_FILE_STUB='OM5_025/ocean_mosaic_v20250916_unpacked/ocean_static.nc' +CMIP6_GOLD_OCEAN_FILE_STUB=None #TODO + +# --------------------------------------------------------------------------- +# MIP-table filtering (used by cmor_config) +# --------------------------------------------------------------------------- +# Table-file suffixes to exclude when scanning a MIP-tables directory for +# variable-entry tables. +EXCLUDED_TABLE_SUFFIXES = [ + 'long_name_overrides', + 'grids', + 'formula_terms', + 'coordinate', + 'cell_measures', +] + + +# --------------------------------------------------------------------------- +# Output / display flags +# --------------------------------------------------------------------------- +# cmor_finder: variable-entry keys to suppress when printing variable info. +DO_NOT_PRINT_LIST = [ + 'comment', + 'ok_min_mean_abs', 'ok_max_mean_abs', + 'valid_min', 'valid_max', +] diff --git a/fremorizer/cmor_finder.py b/fremorizer/cmor_finder.py new file mode 100644 index 0000000..55e6ee4 --- /dev/null +++ b/fremorizer/cmor_finder.py @@ -0,0 +1,205 @@ +""" +fre cmor find +============= + +This module provides tools to find and print information about variables in CMIP6 JSON configuration files. +It is primarily used for inspecting variable entries and generating variable lists for use in FRE CMORization +workflows. + +Functions +--------- +- ``print_var_content(table_config_file, var_name)`` +- ``cmor_find_subtool(json_var_list, json_table_config_dir, opt_var_name)`` +- ``make_simple_varlist(dir_targ, output_variable_list, json_mip_table)`` + +Notes +----- +These utilities are intended to make it easier to inspect and extract variable information from CMIP6 JSON +tables, avoiding the need for manual shell scripting and ad-hoc file inspection. +""" + +import glob +import json +import logging +import os +from pathlib import Path +from typing import Optional, Dict, IO +from .cmor_helpers import get_json_file_data + +from .cmor_constants import DO_NOT_PRINT_LIST + +fre_logger = logging.getLogger(__name__) + +# TODO update for cmip7 if desired +def print_var_content(table_config_file: IO[str], + var_name: str) -> None: + """ + Print information about a specific variable from a given CMIP6 JSON configuration file. + + :param table_config_file: An open file object for a CMIP6 table JSON file. The file should be opened in text mode + :type table_config_file: Input buffer/stream of text, usually output by the open() built-in. See python typing doc + :param var_name: The name of the variable to look for in the configuration file. + :type var_name: str + :raises Exception: If there is an issue reading the JSON content from the file. + :return: None + :rtype: None + + .. note:: Outputs information to the logger at INFO level. + .. note:: If the variable is not found, logs a debug message and returns. + .. note:: Only prints selected fields, omitting any in DO_NOT_PRINT_LIST. + """ + # this function can assume the existence of this was checked in the prev routinue. + proj_table_vars = json.load(table_config_file) + + table_name = None + try: + table_name = proj_table_vars["Header"].get('table_id').split(' ')[1] + except KeyError: + fre_logger.warning("couldn't get header and table_name field") + + if table_name is not None: + fre_logger.info('looking for %s data in table %s!', var_name, table_name) + else: + fre_logger.info('looking for %s data in table %s, but could not find its table_name!', + var_name, table_config_file.name) + + var_content = proj_table_vars.get("variable_entry", {}).get(var_name) + if var_content is None: + fre_logger.debug('variable %s not found in %s, moving on!', var_name, Path(table_config_file.name).name) + return + + fre_logger.info(' variable key: %s', var_name) + for content in var_content: + if content in DO_NOT_PRINT_LIST: + continue + fre_logger.info(' %s: %s', content, var_content[content]) + fre_logger.info('\n') + + +def cmor_find_subtool( json_var_list: Optional[str] = None, + json_table_config_dir: Optional[str] = None, + opt_var_name: Optional[str] = None) -> None: + """ + Find and print information about variables in CMIP6 JSON configuration files in a specified directory. + + :param json_var_list: path to JSON file containing variable names to look up in tables. + :type json_var_list: str or None, optional + :param json_table_config_dir: Directory containing CMIP6 table JSON files. + :type json_table_config_dir: str + :param opt_var_name: Name of a single variable to look up. If None, json_var_list must be provided. + :type opt_var_name: str or None, optional + :raises OSError: If the specified directory does not exist or contains no JSON files. + :raises ValueError: If neither opt_var_name nor json_var_list is provided. + :return: None + :rtype: None + + .. note:: This function is intended as a helper tool for CLI users to quickly inspect variable definitions in + CMIP6 tables. Information is printed via the logger. + """ + if not Path(json_table_config_dir).exists(): + raise OSError(f'ERROR directory {json_table_config_dir} does not exist! exit.') + + fre_logger.info('attempting to find and open files in dir: \n %s ', json_table_config_dir) + json_table_configs = glob.glob(f'{json_table_config_dir}/*.json') + if not json_table_configs: + raise OSError(f'ERROR directory {json_table_config_dir} contains no JSON files, exit.') + fre_logger.info('found content in json_table_config_dir') + + var_list = None + if json_var_list is not None: + with open(json_var_list, "r", encoding="utf-8") as var_list_file: + var_list = json.load(var_list_file) + + if opt_var_name is None and var_list is None: + raise ValueError('ERROR: no opt_var_name given but also no content in variable list!!! exit!') + + if opt_var_name is not None: + fre_logger.info('opt_var_name is not None: looking for only ONE variables worth of info!') + for json_table_config in json_table_configs: + with open(json_table_config, "r", encoding="utf-8") as table_config_file: + print_var_content(table_config_file, opt_var_name) + + elif var_list is not None: + fre_logger.info('opt_var_name is None, and var_list is not None, looking for many variables worth of info!') + for var in var_list: + for json_table_config in json_table_configs: + with open(json_table_config, "r", encoding="utf-8") as table_config_file: + print_var_content(table_config_file, str(var_list[var])) + + +def make_simple_varlist( dir_targ: str, + output_variable_list: Optional[str], + json_mip_table: Optional[str] = None) -> Optional[Dict[str, str]]: + """ + Generate a JSON file containing a list of variable names from NetCDF files in a specified directory. + This function searches for NetCDF files in the given directory, or a subdirectory, "ts/monthly/5yr", + if not already included. It then extracts variable names from the filenames, and writes these variable + names to a JSON file. + + :param dir_targ: The target directory to search for NetCDF files. + :type dir_targ: str + :param output_variable_list: The path to the output JSON file where the variable list will be saved. + :type output_variable_list: str + :param json_mip_table: target table for making the var list. found variables are included if they are in the table + :type json_mip_table: str + :raises OSError: if the outputfile cannot be written + :return: Dictionary of variable names (keys == values), or None if no files are found or an error occurs + :rtype: dict or None + + .. note:: Assumes NetCDF filenames are of the form: ...nc + .. note:: Variable name is assumed to be the second-to-last component when split by periods. + .. note:: Logs a warning if only one file is found. + + .. warning:: Logs errors if no files are found in the directory or if no files match the expected pattern. + + """ + # if the variable is in the filename, it's likely delimited by another period. + all_nc_files = glob.glob(os.path.join(dir_targ, "*.*.nc")) + if not all_nc_files: + fre_logger.error("No files found in the directory.") #uncovered + return None + + if len(all_nc_files) == 1: + fre_logger.warning("Warning: Only one file found matching the pattern.") + + fre_logger.info("Files found matching pattern. Number of files: %d", len(all_nc_files)) + + mip_vars = None + if json_mip_table is not None: + try: + # read in mip vars to check against later + fre_logger.debug('attempting to read in variable entries in specified mip table') + full_mip_vars_list=get_json_file_data(json_mip_table)["variable_entry"].keys() + + except Exception as exc: + raise Exception( 'problem opening mip table and getting variable entry data.' + f'exc = {exc}') from exc + + fre_logger.debug('attempting to make mip variable list') + mip_vars=[ key.split('_')[0] for key in full_mip_vars_list ] + fre_logger.debug('mip vars extracted for comparison when making var list: %s', mip_vars) + + # Build a deduplicated dict of variable names extracted from all filenames across + # all datetimes. Assigning to a dict naturally deduplicates while preserving + # first-seen insertion order (Python 3.7+). + var_list: Dict[str, str] = {} + for targetfile in all_nc_files: + var_name=os.path.basename(targetfile).split('.')[-2] + if mip_vars is not None and var_name not in mip_vars: + continue + var_list[var_name] = var_name + + if not var_list: + fre_logger.warning('WARNING: no variables in target mip table found, or no matching pattern,' + ' or not enough info in the filenames (i am expecting FRE-bronx like filenames)') + return None + + # Write the variable list to the output JSON file + if output_variable_list is not None: + try: + fre_logger.info('writing output variable list, %s', list(var_list.keys())) + with open(output_variable_list, 'w', encoding='utf-8') as f: + json.dump(var_list, f, indent=4) + except Exception as exc: + raise OSError('output variable list created but cannot be written') from exc + return var_list diff --git a/fremorizer/cmor_helpers.py b/fremorizer/cmor_helpers.py new file mode 100644 index 0000000..a7e6c90 --- /dev/null +++ b/fremorizer/cmor_helpers.py @@ -0,0 +1,784 @@ +""" +fre.cmor helper functions +========================= + +This module provides helper functions for the CMORization workflow in the FRE (Flexible Runtime Environment) +CLI, specifically for use in the cmor_mixer submodule. The utilities here support a variety of common +tasks including: + +- Logging and min/max value inspection for masked arrays. +- Extraction and manipulation of variables from netCDF4 datasets. +- File path and directory utilities tailored to FRE conventions. +- Construction of boundary arrays for vertical levels. +- Extraction and filtering of ISO datetime ranges from filenames. +- Detection of ocean grid conventions in datasets. +- Determination of vertical dimension names in datasets. +- Creation of temporary output directories for CMOR products. +- Reading and updating experiment configuration JSON files. + +Functions +--------- +- ``print_data_minmax(ds_variable, desc)`` +- ``from_dis_gimme_dis(from_dis, gimme_dis)`` +- ``find_statics_file(bronx_file_path)`` +- ``create_lev_bnds(bound_these, with_these)`` +- ``get_iso_datetime_ranges(var_filenames, iso_daterange_arr, start, stop)`` +- ``check_dataset_for_ocean_grid(ds)`` +- ``get_vertical_dimension(ds, target_var)`` +- ``create_tmp_dir(outdir, json_exp_config)`` +- ``get_json_file_data(json_file_path)`` +- ``update_grid_and_label(json_file_path, new_grid_label, new_grid, new_nom_res, output_file_path)`` +- ``update_calendar_type(json_file_path, new_calendar_type, output_file_path)`` +- ``check_path_existence(some_path)`` +- ``iso_to_bronx_chunk(cmor_chunk_in)`` +- ``conv_mip_to_bronx_freq(cmor_table_freq)`` +- ``get_bronx_freq_from_mip_table(json_table_config)`` +- ``filter_brands(brands, target_var, mip_var_cfgs, has_time_bnds, input_vert_dim)`` + +Notes +----- +These functions aim to encapsulate frequently repeated logic in the CMOR workflow, improving code +readability, maintainability, and robustness. +""" + +import glob +import json +import logging +import os +from pathlib import Path +import shutil +import subprocess +from typing import Optional, List, Union + +import numpy as np +from netCDF4 import Dataset, Variable + +from .cmor_constants import ( ARCHIVE_GOLD_DATA_DIR, CMIP7_GOLD_OCEAN_FILE_STUB, CMIP6_GOLD_OCEAN_FILE_STUB, + INPUT_TO_MIP_VERT_DIM ) + +fre_logger = logging.getLogger(__name__) + + +def print_data_minmax( ds_variable: Optional[np.ma.core.MaskedArray] = None, + desc: Optional[str] = None) -> None: + """ + Log the minimum and maximum values of a numpy MaskedArray along with a description. + + :param ds_variable: The data array whose min/max is to be logged. + :type ds_variable: numpy.ma.core.MaskedArray, optional + :param desc: Description of the data. + :type desc: str, optional + + :return: None + :rtype: None + + .. note:: If the data cannot be logged, a warning is issued. + """ + try: + fre_logger.info('info for \n desc = %s \n %s', desc, type(ds_variable)) + fre_logger.info('%s < %s < %s', ds_variable.min(), desc, ds_variable.max()) + except Exception: + fre_logger.warning('could not print min/max entries for desc = %s', desc) + + +def from_dis_gimme_dis( from_dis: Dataset, + gimme_dis: str) -> Optional[np.ndarray]: + """ + Retrieve and return a copy of a variable from a netCDF4.Dataset-like object. + + :param from_dis: The source dataset object. + :type from_dis: netCDF4.Dataset + :param gimme_dis: The variable name to extract from the dataset. + :type gimme_dis: str + :return: A copy of the requested variable's data, or None if not found. + :rtype: np.ndarray or None + + .. note:: Logs a warning if the variable is not found. The name comes from a hypothetical pronunciation of 'ds', + the common monniker for a netCDF4.Dataset object. + """ + try: + return from_dis[gimme_dis][:].copy() + except Exception: + fre_logger.warning('I am sorry, I could not not give you this: %s\n returning None!\n', gimme_dis) + return None + + +def find_gold_ocean_statics_file(put_copy_here: Optional[str] = None) -> Optional[str]: + """ + Locate (and if necessary copy) the gold-standard OM5_025 ocean_static.nc file + from the GFDL archive into a user-writable directory. + + :param put_copy_here: Directory root under which a mirror of the archive + sub-path will be created and the file copied into. + :type put_copy_here: str or None + :return: Absolute path to the local working copy of ocean_static.nc, + or None if the file could not be obtained. + :rtype: str or None + + .. note:: The archive path is hard-coded to the OM5_025 dataset on GFDL systems. + """ + archive_gold_file = ( + f'{ARCHIVE_GOLD_DATA_DIR}/{CMIP7_GOLD_OCEAN_FILE_STUB}' + #f'{ARCHIVE_GOLD_DATA_DIR}/OM5_025/ocean_mosaic_v20250916_unpacked/ocean_static.nc' + ) + fre_logger.debug('ARCHIVE_GOLD_DATA_DIR=%s', ARCHIVE_GOLD_DATA_DIR) + fre_logger.debug('archive_gold_file=%s', archive_gold_file) + + if not Path(archive_gold_file).exists(): + fre_logger.error('ERROR gold archive file does not exist: %s', archive_gold_file) + fre_logger.error('ERROR this file should probably exist.') + fre_logger.warning('WARNING i will fallback to using buggy ocean_statics' + ' files in pp directories out of desperation') + return None + + + if put_copy_here is None: + fre_logger.warning('put_copy_here is None, cannot stage gold ocean statics file') + return None + + # mirror the archive sub-path under put_copy_here + # e.g. /archive/gold/datasets/OM5_025/… -> datasets/OM5_025/… + #try: + new_dir_tree = CMIP7_GOLD_OCEAN_FILE_STUB # '/'.join(archive_gold_file.split('/')[3:]) + fre_logger.debug('new_dir_tree=%s', new_dir_tree) + #except Exception: + # fre_logger.error('could not derive sub-path from archive_gold_file') + # return None + + working_copy_dir = f'{put_copy_here}/{Path(new_dir_tree).parent}' + Path(working_copy_dir).mkdir(parents=True, exist_ok=True) + working_copy = f'{working_copy_dir}/{Path(archive_gold_file).name}' + + # guard: if a stale directory exists where the file should be (from a prior buggy mkdir), + # remove it so the copy can succeed + if Path(working_copy).exists(): + if Path(working_copy).is_dir(): + fre_logger.warning('prior buggy mkdir suspected- removing directory instead of the expected file') + shutil.rmtree(working_copy) + fre_logger.warning('dir removed, moving on') + else: + fre_logger.warning('a previous copy of the ocean statics file exists, not re-copying!') + return working_copy + + if not Path(working_copy).is_file(): + fre_logger.info('copying archived golden statics file to\n %s', working_copy) + try: + subprocess.run(['cp', archive_gold_file, working_copy], shell=False, check=True) + except subprocess.CalledProcessError as exc: + fre_logger.warning('cp of gold statics file failed: %s', exc) + return None + + if Path(working_copy).is_file(): + fre_logger.info('gold ocean statics file available at %s', working_copy) + return working_copy + + fre_logger.warning('gold ocean statics file not available after copy attempt') + return None + +# note, the awkward spacing of the docstring below is for the way sphinx renders reStructuredText, do not change! +def find_statics_file( bronx_file_path: str) -> Optional[str]: + """ + Attempt to find the corresponding statics file given the path to a FRE-bronx output file. The code assumes + the output file is in a FRE-bronx directory structure when trying to access the statics file. The structure is + mocked in this package within the `fre/tests/test_files/ascii_files/mock_archive` directory structure. `cd`'ing + there and using the command `tree` will reveal the mocked directory structure, something like: + + + //-/ + + └── pp + + ├── component + + ├── realm_frequency.static.nc + + └── ts + + └── frequency + + └── chunk_size + + └── component.YYYYMM-YYYYMM.var.nc + + + :param bronx_file_path: File path to use as a reference for statics file location. + :type bronx_file_path: str + :return: Path to the statics file if found, else None. + :rtype: str or None + + .. note:: The function searches upward in the directory structure until it finds a 'pp' directory, then globs + for '*static*.nc' files. + """ + bronx_file_path_elem = bronx_file_path.split('/') + num_elem = len(bronx_file_path_elem) + fre_logger.debug('bronx_file_path_elem = \n%s\n', bronx_file_path_elem) + while bronx_file_path_elem[num_elem-2] != 'pp': + bronx_file_path_elem.pop() + num_elem = num_elem-1 + statics_path = '/'.join(bronx_file_path_elem) + fre_logger.debug('going to glob the following path for a statics file: \n%s\n', statics_path) + fre_logger.debug('the call is going to be:') + fre_logger.debug(f"\n glob.glob({statics_path+'/*static*.nc'}) \n") + + statics_file_glob = glob.glob(statics_path+'/*static*.nc') # update to use component TODO + fre_logger.debug('the output glob looks like: %s', statics_file_glob) + if len(statics_file_glob) == 1: + return statics_file_glob[0] + + fre_logger.warning('no statics file found, returning None') + return None + + +def create_lev_bnds( bound_these: Variable = None, + with_these: Variable = None) -> np.ndarray: + """ + Create a vertical level bounds array for a set of levels. + + :param bound_these: netCDF4 Variable with a numpy array representing vertical levels + :type bound_these: netCDF4.Variable + :param with_these: netCDF4 Variable with a numpy array representing level bounds, one longer than bound_these + :type with_these: netCDF4.Variable + :raises ValueError: If the length of with_these is not len(bound_these) + 1. + :return: Array of shape (len(bound_these), 2), where each row gives the bounds for a level. + :rtype: np.ndarray + + .. note:: Logs debug information about the input and output arrays. + """ + if len(with_these) != (len(bound_these) + 1): + raise ValueError('failed creating bnds on-the-fly :-(') + fre_logger.debug('bound_these = \n%s', bound_these) + fre_logger.debug('with_these = \n%s', with_these) + + the_bnds = np.arange(len(bound_these)*2).reshape(len(bound_these), 2) + for i in range(0, len(bound_these)): + the_bnds[i][0] = with_these[i] + the_bnds[i][1] = with_these[i+1] + fre_logger.info('the_bnds = \n%s', the_bnds) + return the_bnds + + +def get_iso_datetime_ranges( var_filenames: List[str], + iso_daterange_arr: Optional[List[str]] = None, + start: Optional[str] = None, + stop: Optional[str] = None) -> None: + """ + Extract and append ISO datetime ranges from filenames, filtered by start/stop years if specified. + + :param var_filenames: Filenames, some of which contain ISO datetime ranges (e.g. 'YYYYMMDD-YYYYMMDD'). + :type var_filenames: list of str + :param iso_daterange_arr: List to append found datetime ranges to; modified in-place. + :type iso_daterange_arr: list of str + :param start: Start year in 'YYYY' format; only ranges within/after this year are included. + :type start: str, optional + :param stop: Stop year in 'YYYY' format; only ranges within/before this year are included. + :type stop: str, optional + :raises ValueError: If iso_daterange_arr is not provided or if no datetime ranges are found. + :return: None + :rtype: None + + .. note:: This function modifies iso_daterange_arr in-place. + """ + fre_logger.debug('start = %s', start) + fre_logger.debug('stop = %s', stop) + start_stop_filter = False + stop_yr_int, start_yr_int = None, None + if start is not None and len(start) == 4: + start_yr_int = int(start) + start_stop_filter = True + if stop is not None and len(stop) == 4: + stop_yr_int = int(stop) + start_stop_filter = True + fre_logger.debug('start_yr_int = %s', start_yr_int) + fre_logger.debug(' stop_yr_int = %s', stop_yr_int) + + if iso_daterange_arr is None: + raise ValueError( + 'this function requires the list one desires to fill with datetime ranges from filenames') + + for filename in var_filenames: + fre_logger.debug('filename = %s', filename) + iso_daterange = filename.split(".")[-3] # '????????-????????' + fre_logger.debug('iso_daterange = %s', iso_daterange) + + if start_stop_filter: + iso_datetimes = iso_daterange.split('-') + fre_logger.debug('iso_datetimes = %s', iso_datetimes) + if start is not None and int(iso_datetimes[0][0:4]) < start_yr_int: + continue + if stop is not None and int(iso_datetimes[1][0:4]) > stop_yr_int: + continue + + if iso_daterange not in iso_daterange_arr: + iso_daterange_arr.append(iso_daterange) + + iso_daterange_arr.sort() + + if len(iso_daterange_arr) < 1: + raise ValueError('iso_daterange_arr has length 0! i need to find at least one datetime range!') + + +def check_dataset_for_ocean_grid( ds: Dataset) -> bool: + """ + Check if a netCDF4.Dataset uses an ocean grid (i.e., contains 'xh' or 'yh' variables). + + :param ds: Dataset to be checked. + :type ds: netCDF4.Dataset + :return: True if ocean grid variables are present, otherwise False. + :rtype: bool + + .. note:: Logs a warning if an ocean grid is detected. + """ + ds_var_keys = list(ds.variables.keys()) + uses_ocean_grid = any(["xh" in ds_var_keys, "yh" in ds_var_keys]) + if uses_ocean_grid: + fre_logger.warning( + "\n----------------------------------------------------------------------------------\n" + " 'xh' found in var_list: ocean grid req'd\n" + " sometimes i don't cmorize right! check me!\n" + "----------------------------------------------------------------------------------\n" + ) + return uses_ocean_grid + + +def get_vertical_dimension( ds: Dataset, + target_var: str) -> Union[str, int]: + """ + Determine the vertical dimension for a variable in a netCDF4.Dataset. + + :param ds: Dataset containing variables. + :type ds: netCDF4.Dataset + :param target_var: Name of the variable to inspect. + :type target_var: str + :return: Name of the vertical dimension if found, otherwise 0. + :rtype: str or int + + .. note:: Returns 0 if no vertical dimension is detected. + """ + vert_dim = 0 + for name, variable in ds.variables.items(): + if name != target_var: + continue + dims = variable.dimensions + for dim in dims: + if dim.lower() == 'landuse': + vert_dim = dim + break + if not (ds[dim].axis and ds[dim].axis == "Z"): + continue + vert_dim = dim + return vert_dim + + +def create_tmp_dir( outdir: str, + json_exp_config: Optional[str] = None) -> str: + """ + Create a temporary directory for output, possibly informed by a JSON experiment config. + + :param outdir: Base output directory. + :type outdir: str + :param json_exp_config: Path to a JSON config file with an "outpath" key. + :type json_exp_config: str, optional + :raises OSError: If the temporary directory cannot be created. + :return: Path to the created temporary directory. + :rtype: str + + .. note:: If json_exp_config is provided and contains "outpath", a subdirectory is also created. + """ + outdir_from_exp_config = None + if json_exp_config is not None: + with open(json_exp_config, "r", encoding="utf-8") as table_config_file: + try: + outdir_from_exp_config = json.load(table_config_file)["outpath"] + except Exception: + fre_logger.warning( + 'could not read outdir from json_exp_config. the cmor module will throw a toothless warning') + + tmp_dir = str(Path("{}/CMOR_tmp/".format(outdir)).resolve()) + try: + os.makedirs(tmp_dir, exist_ok=True) + if outdir_from_exp_config is not None: + fre_logger.info('attempting to create %s dir in tmp_dir targ', outdir_from_exp_config) + try: + os.makedirs(tmp_dir + '/' + outdir_from_exp_config, exist_ok=True) + except Exception: + fre_logger.info('attempting to create %s dir in tmp_dir targ did not work', outdir_from_exp_config) + fre_logger.info('... attempt to avoid a toothless cmor warning failed... moving on') + except Exception as exc: + raise OSError('problem creating tmp output directory {}. stop.'.format(tmp_dir)) from exc + + return tmp_dir + + +def get_json_file_data( json_file_path: Optional[str] = None) -> dict: + """ + Load and return the contents of a JSON file. + + :param json_file_path: Path to the JSON file. + :type json_file_path: str + :raises FileNotFoundError: If the file cannot be opened. + :return: Parsed data from the JSON file. + :rtype: dict + """ + try: + with open(json_file_path, "r", encoding="utf-8") as json_config_file: + return json.load(json_config_file) + except Exception as exc: + raise FileNotFoundError( + 'ERROR: json_file_path file cannot be opened.\n' + ' json_file_path = {}'.format(json_file_path) + ) from exc + + +def update_grid_and_label( json_file_path: str, + new_grid_label: str, + new_grid: str, + new_nom_res: str, + output_file_path: Optional[str] = None) -> None: + """ + Update the "grid_label", "grid", and "nominal_resolution" fields in a JSON experiment config. + + :param json_file_path: Path to the input JSON file. + :type json_file_path: str + :param new_grid_label: New value for the "grid_label" field. + :type new_grid_label: str + :param new_grid: New value for the "grid" field. + :type new_grid: str + :param new_nom_res: New value for the "nominal_resolution" field. + :type new_nom_res: str + :param output_file_path: Path to save the updated JSON file. If None, overwrites the original file. + :type output_file_path: str, optional + :raises FileNotFoundError: If the input JSON file does not exist. + :raises KeyError: If a required field is not found in the JSON file. + :raises ValueError: If any input value is None. + :raises json.JSONDecodeError: If the JSON file cannot be decoded. + :return: None + :rtype: None + + .. note:: The function logs before and after values, and overwrites the input file unless an output path is given. + """ + if None in [new_grid_label, new_grid, new_nom_res]: + fre_logger.error( + 'grid/grid_label/nom_res updating requested for exp_config file, but one of them is None\n' + 'bailing...!') + raise ValueError + + try: + with open(json_file_path, "r", encoding="utf-8") as file: + data = json.load(file) + + try: + fre_logger.info('Original "grid": %s', data["grid"]) + data["grid"] = new_grid + fre_logger.info('Updated "grid": %s', data["grid"]) + except KeyError as e: + fre_logger.error("Failed to update 'grid': %s", e) + raise KeyError("Error while updating 'grid'. Ensure the field exists and is modifiable.") from e + + try: + fre_logger.info('Original "grid_label": %s', data["grid_label"]) + data["grid_label"] = new_grid_label + fre_logger.info('Updated "grid_label": %s', data["grid_label"]) + except KeyError as e: + fre_logger.error("Failed to update 'grid_label': %s", e) + raise KeyError("Error while updating 'grid_label'. Ensure the field exists and is modifiable.") from e + + try: + fre_logger.info('Original "nominal_resolution": %s', data["nominal_resolution"]) + data["nominal_resolution"] = new_nom_res + fre_logger.info('Updated "nominal_resolution": %s', data["nominal_resolution"]) + except KeyError as e: + fre_logger.error("Failed to update 'nominal_resolution': %s", e) + raise KeyError("Error updating 'nominal_resolution'. Ensure the field exists and is modifiable.") from e + + output_file_path = output_file_path or json_file_path + + with open(output_file_path, "w", encoding="utf-8") as file: + json.dump(data, file, indent=4) + + fre_logger.info('Successfully updated fields and saved to %s', output_file_path) + + except FileNotFoundError: + fre_logger.error("The file '%s' does not exist.", json_file_path) + raise + except json.JSONDecodeError: + fre_logger.error("Failed to decode JSON from the file '%s'.", json_file_path) + raise + except Exception as e: + fre_logger.error("An unexpected error occurred: %s", e) + raise + + +def update_calendar_type( json_file_path: str, + new_calendar_type: str, + output_file_path: Optional[str] = None) -> None: + """ + Update the "calendar" field in a JSON experiment config file. + + :param json_file_path: Path to the input JSON file. + :type json_file_path: str + :param new_calendar_type: New value for the "calendar" field. + :type new_calendar_type: str + :param output_file_path: Path to save the updated JSON file. If None, overwrites the original file. + :type output_file_path: str, optional + :raises FileNotFoundError: If the input JSON file does not exist. + :raises KeyError: If the "calendar" field is not found in the JSON file. + :raises ValueError: If new_calendar_type is None. + :raises json.JSONDecodeError: If the JSON file cannot be decoded. + :return: None + :rtype: None + + .. note:: The function logs before and after values, and overwrites the input file unless an output path is given. + """ + if new_calendar_type is None: + fre_logger.error( + 'calendar_type updating requested for exp_config file, but one of them is None\n' + 'bailing...!') + raise ValueError + + try: + with open(json_file_path, "r", encoding="utf-8") as file: + data = json.load(file) + + try: + fre_logger.info('Original "calendar": %s', data["calendar"]) + data["calendar"] = new_calendar_type + fre_logger.info('Updated "calendar": %s', data["calendar"]) + except KeyError as e: + fre_logger.error("Failed to update 'calendar': %s", e) + raise KeyError("Error while updating 'calendar'. Ensure the field exists and is modifiable.") from e + + output_file_path = output_file_path or json_file_path + + with open(output_file_path, "w", encoding="utf-8") as file: + json.dump(data, file, indent=4) + + fre_logger.info('Successfully updated fields and saved to %s', output_file_path) + + except FileNotFoundError: + fre_logger.error("The file '%s' does not exist.", json_file_path) + raise + except json.JSONDecodeError: + fre_logger.error("Failed to decode JSON from the file '%s'.", json_file_path) + raise + except Exception as e: + fre_logger.error("An unexpected error occurred: %s", e) + raise + +def check_path_existence(some_path: str): + """ + Check if the given path exists, raising FileNotFoundError if not. + + :param some_path: A string representing a filesystem path (relative or absolute). + :type some_path: str + :raises FileNotFoundError: If the path does not exist. + """ + if not Path(some_path).exists(): + raise FileNotFoundError(f'does not exist: {some_path}') + +def iso_to_bronx_chunk(cmor_chunk_in: str) -> str: + """ + Convert an ISO8601 duration string (e.g., 'P5Y') to FRE-bronx-style chunk string (e.g., '5yr'). + + :param cmor_chunk_in: ISO8601 formatted string representing a time interval (must start with 'P' and end with 'Y'). + :type cmor_chunk_in: str + :raises ValueError: If the input does not follow the expected ISO format. + :return: FRE-bronx chunk string. + :rtype: str + """ + fre_logger.debug('cmor_chunk_in = %s', cmor_chunk_in) + if cmor_chunk_in[0] == 'P' and cmor_chunk_in[-1] == 'Y': + bronx_chunk = f'{cmor_chunk_in[1:-1]}yr' + else: + raise ValueError('problem with converting to bronx chunk from the cmor chunk. check cmor_yamler.py') + fre_logger.debug('bronx_chunk = %s', bronx_chunk) + return bronx_chunk + +def conv_mip_to_bronx_freq(cmor_table_freq: str) -> Optional[str]: + """ + Convert a MIP table frequency string to its FRE-bronx equivalent using a lookup table. + + :param cmor_table_freq: Frequency string as found in a MIP table (e.g., 'mon', 'day', 'yr', etc.). + :type cmor_table_freq: str + :raises KeyError: If the frequency string is not recognized as valid. + :return: FRE-bronx frequency string, or None if not mappable. + :rtype: str or None + """ + cmor_to_bronx_dict = { + "1hr" : "1hr", + "1hrCM" : None, + "1hrPt" : None, + "3hr" : "3hr", + "3hrPt" : None, + "6hr" : "6hr", + "6hrPt" : None, + "day" : "daily", + "dec" : None, + "fx" : None, + "mon" : "monthly", + "monC" : None, + "monPt" : None, + "subhrPt": None, + "yr" : "annual", + "yrPt" : None + } + bronx_freq = cmor_to_bronx_dict.get(cmor_table_freq) + if bronx_freq is None: + fre_logger.warning('MIP table frequency = %s does not have a FRE-bronx equivalent', cmor_table_freq) + if cmor_table_freq not in cmor_to_bronx_dict.keys(): + raise KeyError(f'MIP table frequency = "{cmor_table_freq}" is not a valid MIP frequency') + return bronx_freq + +def get_bronx_freq_from_mip_table(json_table_config: str) -> str: + """ + Extract the frequency of data from a CMIP MIP table (JSON), returning its FRE-bronx equivalent. + + :param json_table_config: Path to a JSON MIP table file with 'variable_entry' metadata. + :type json_table_config: str + :raises KeyError: If the frequency cannot be found or mapped. + :return: FRE-bronx frequency string. + :rtype: str + """ + table_freq = None + with open(json_table_config, 'r', encoding='utf-8') as table_config_file: + table_config_data = json.load(table_config_file) + for var_entry in table_config_data['variable_entry']: + try: + table_freq = table_config_data['variable_entry'][var_entry]['frequency'] + break + except Exception as exc: + raise KeyError('no frequency in table under variable_entry. this may be a CMIP7 table.') from exc + + bronx_freq = conv_mip_to_bronx_freq(table_freq) + return bronx_freq + +#def update_outpath( json_file_path: str, +# outpath: str, +# output_file_path: Optional[str] = None) -> None: +# """ +# Update the "outpath" field in a JSON experiment config file. +# +# :param json_file_path: Path to the input JSON file. +# :type json_file_path: str +# :param outpath: key in input experiment config for managing target output directory, required +# :type outpath: str +# :param output_file_path: path to write the updated experiment config file to, if desired. +# :type output_file_path: str, optional +# """ +# +# if None in [json_file_path, outpath]: +# fre_logger.error( +# 'a required input argument is None\n' +# 'bailing...!') +# raise ValueError +# +# try: +# with open(json_file_path, "r", encoding="utf-8") as file: +# data = json.load(file) +# +# try: +# fre_logger.info('Original "outpath": %s', data["outpath"]) +# data["outpath"] = outpath +# fre_logger.info('Updated "outpath": %s', data["outpath"]) +# except KeyError as e: +# fre_logger.error("Failed to update 'outpath': %s", e) +# raise KeyError("Error while updating 'outpath'. Ensure the field exists and is modifiable.") from e +# +# output_file_path = output_file_path or json_file_path +# +# with open(output_file_path, "w", encoding="utf-8") as file: +# json.dump(data, file, indent=4) +# +# fre_logger.info('Successfully updated fields and saved to %s', output_file_path) +# +# except FileNotFoundError: +# fre_logger.error("The file '%s' does not exist.", json_file_path) +# raise +# except json.JSONDecodeError: +# fre_logger.error("Failed to decode JSON from the file '%s'.", json_file_path) +# raise +# except Exception as e: +# fre_logger.error("An unexpected error occurred: %s", e) +# raise + + +def filter_brands( brands: list, + target_var: str, + mip_var_cfgs: dict, + has_time_bnds: bool, + input_vert_dim: Union[str, int] ) -> str: + """ + Disambiguate multiple CMIP7 variable brands by comparing input data + properties against each candidate brand's MIP dimension list. + + Two filters are applied in sequence: + + 1. **Time type**: The presence or absence of time bounds in the input data + is compared to whether the brand's MIP dimensions contain ``time`` + (time-mean, has bounds) or ``time1`` (instantaneous, no bounds). + 2. **Vertical coordinate**: The input data's vertical dimension name is + mapped to the corresponding MIP dimension name (via + ``INPUT_TO_MIP_VERT_DIM``) and brands whose MIP dimensions do not + include it are excluded. + + :param brands: List of candidate brand strings to filter. + :type brands: list[str] + :param target_var: The base variable name (before the brand suffix). + :type target_var: str + :param mip_var_cfgs: The full MIP table config dict (must contain + ``"variable_entry"``). + :type mip_var_cfgs: dict + :param has_time_bnds: Whether the input dataset contains ``time_bnds``. + :type has_time_bnds: bool + :param input_vert_dim: The vertical dimension name from the input dataset, + or ``0`` if no vertical dimension is present. + :type input_vert_dim: str or int + :raises ValueError: If zero or more than one brand survives filtering. + :return: The single brand string that survived disambiguation. + :rtype: str + """ + # map input vertical dim to MIP equivalent + expected_mip_vert = None + if input_vert_dim != 0: + expected_mip_vert = INPUT_TO_MIP_VERT_DIM.get( + input_vert_dim.lower(), input_vert_dim.lower()) + + filtered_brands = [] + for brand in brands: + mip_key = f'{target_var}_{brand}' + mip_dims = mip_var_cfgs["variable_entry"][mip_key]["dimensions"] + + # time filter + if has_time_bnds and 'time1' in mip_dims: + fre_logger.debug('filtering out brand %s: MIP dims contain time1 ' + 'but input data has time_bnds', brand) + continue + if not has_time_bnds and 'time' in mip_dims and 'time1' not in mip_dims: + fre_logger.debug('filtering out brand %s: MIP dims contain time (mean) ' + 'but input data lacks time_bnds', brand) + continue + + # vertical filter + if expected_mip_vert is not None and expected_mip_vert not in mip_dims: + fre_logger.debug('filtering out brand %s: expected MIP vert dim %s ' + 'not found in %s', brand, expected_mip_vert, mip_dims) + continue + + filtered_brands.append(brand) + + if len(filtered_brands) == 1: + fre_logger.info('cmip7 brand disambiguation successful, selected brand: %s', + filtered_brands[0]) + return filtered_brands[0] + + if len(filtered_brands) == 0: + fre_logger.error('cmip7 brand disambiguation eliminated all candidates ' + 'from %s', brands) + raise ValueError( + f'multiple brands {brands} found for {target_var}, ' + f'but none survived disambiguation filtering') + + fre_logger.error('cmip7 brand disambiguation could not resolve between ' + '%s', filtered_brands) + raise ValueError( + f'multiple brands {filtered_brands} remain for {target_var} after ' + f'disambiguation \u2014 cannot determine which brand to use') diff --git a/fremorizer/cmor_mixer.py b/fremorizer/cmor_mixer.py new file mode 100644 index 0000000..43aa55d --- /dev/null +++ b/fremorizer/cmor_mixer.py @@ -0,0 +1,1085 @@ +""" +FRE / CMOR Metadata Mixing and Rewriting (CMORization) +====================================================== + +This module provides routines which rewrite post-processed FRE/FMS model output in a community-driven, standardized way. +This module relies heavily on PCMDI's CMOR module and it's python API. It is the core implementation for +``fre cmor run`` operations- mixing and matching GFDL's and FRE's conventions to CMOR's expectations, so that +participation in model-intercomparison projects may be eased. For more usage details, see the project README.md, the +FRE documentation, and PCMDI's CMOR module documentation available at https://cmor.llnl.gov/. + +This module currently follows a composite pattern, analogous to nested/russian doll, where a large piece contains a +smaller piece which contains another. ``fre cmor run`` leads directly to ``cmor_run_subtool``, which calls +``cmorize_all_variables_in_dir`` once, which calls ``cmorize_all_variables_in_dir`` once, which calls +``cmorize_target_var_files`` once per variable in a variable list, and calls ``rewrite_netcdf_file_var`` once per found +datetime for a given variable. Functions within ``cmor_helpers`` assist with the CMORization process. + +Functions +--------- +- ``rewrite_netcdf_file_var(...)`` +- ``cmorize_target_var_files(...)`` +- ``cmorize_all_variables_in_dir(...)`` +- ``cmor_run_subtool(...)`` + +.. note:: The name "mixer" comes from a conversation between Chris Blanton, the original code author (Sergey Nikonov), + and the next author/maintainer, Ian Laflotte, in 2022. Chris wanted to change the name, and Sergey kind of + enjoyed the original CMORCommander.py, and so did not have any suggestions. Ian, whom was very new and knew + nothing, suggested "cmor mixer", not truly understanding why. Chris and Sergey decided to go with it. +""" + + +import getpass +import glob +import json +import logging +import os +from pathlib import Path +import shutil +import subprocess +from typing import Optional, List, Dict, Any + +import cmor +import numpy as np +import netCDF4 as nc + +from .cmor_helpers import ( print_data_minmax, from_dis_gimme_dis, find_statics_file, create_lev_bnds, + get_iso_datetime_ranges, check_dataset_for_ocean_grid, get_vertical_dimension, + create_tmp_dir, get_json_file_data, update_grid_and_label, #update_outpath, + update_calendar_type, find_gold_ocean_statics_file, filter_brands ) +from .cmor_constants import ( ACCEPTED_VERT_DIMS, NON_HYBRID_SIGMA_COORDS, ALT_HYBRID_SIGMA_COORDS, + DEPTH_COORDS, CMOR_NC_FILE_ACTION, CMOR_VERBOSITY, + CMOR_EXIT_CTL, CMOR_MK_SUBDIRS, CMOR_LOG ) + +fre_logger = logging.getLogger(__name__) + +def rewrite_netcdf_file_var( mip_var_cfgs: dict = None, + local_var: str = None, + netcdf_file: str = None, + target_var: str = None, + json_exp_config: str = None, + json_table_config: str = None, + prev_path: Optional[str] = None ) -> str: + """ + Rewrite the input NetCDF file for a target variable in a CMIP-compliant manner and write output using CMOR. + + :param mip_var_cfgs: Variable table, as loaded from the MIP table JSON config. + :type mip_var_cfgs: dict + :param local_var: Variable name used for finding files locally. + :type local_var: str + :param netcdf_file: Path to the input NetCDF file to be CMORized. + :type netcdf_file: str + :param target_var: Name of the variable to be processed. + :type target_var: str + :param json_exp_config: Path to experiment configuration JSON file (for dataset metadata). + :type json_exp_config: str + :param json_table_config: Path to MIP table JSON file. + :type json_table_config: str + :param prev_path: Path to previous file (used for finding statics file for tripolar grids). + :type prev_path: str, optional + :raises ValueError: If unsupported vertical dimensions or inconsistent grid dimensions are found. + :raises FileNotFoundError: If required statics file for tripolar ocean grid is missing. + :raises Exception: For other errors in the metadata, file IO, or CMOR calls. + :return: Absolute path to the output file written by cmor.close. + :rtype: str + + .. note:: This function performs extensive setup of axes and metadata, and conditionally handles tripolar + ocean grids. + """ + fre_logger.info("input data:") + fre_logger.info(" local_var = %s", local_var) + fre_logger.info(" target_var = %s", target_var) + + # open the input file + fre_logger.info("opening %s", netcdf_file) + ds = nc.Dataset(netcdf_file, 'r+') + + # read the input variable data + fre_logger.info('attempting to read variable data, %s', target_var) + var = from_dis_gimme_dis(from_dis=ds, gimme_dis=target_var) + + ## var type + #var_dtype = var.dtype + + # var missing_value, in numpy masked_array land, called the fill_value + var_missing_val = var.fill_value + + # grab var_dim + var_dim = len(var.shape) + fre_logger.info("var_dim = %d, local_var = %s", var_dim, local_var) + + # CMORizing ocean grids are implemented only for scalar quantities valued at the central T/h-point of the grid cell. + # https://en.wikipedia.org/wiki/Arakawa_grids heavily consulted for this work. + # we also need to do: + # - ice tripolar cases + # - vector cases (quantities valued on edges in B/C/D for ocean and ice) + # - probably others that i cannot currently fathom but will bump into. + fre_logger.info('checking input netcdf file for oceangrid condition') + uses_ocean_grid = check_dataset_for_ocean_grid(ds) + if uses_ocean_grid: + fre_logger.warning( + 'cmor_mixer suspects this is ocean data, being reported on \n' + ' native tripolar grid. i may treat this differently than other files!' + ) + + # check for cmip7 case and extract possible brands here + var_brand = None + exp_cfg_mip_era = get_json_file_data(json_exp_config)['mip_era'].upper() + if exp_cfg_mip_era == 'CMIP7': + brands = [] + for mip_var in mip_var_cfgs["variable_entry"].keys(): + if all([ target_var == mip_var.split('_')[0], + var_dim == len(mip_var_cfgs["variable_entry"][mip_var]['dimensions']) ]): + brands.append(mip_var.split('_')[1]) + + if len(brands)>0: + if len(brands)==1: + var_brand=brands[0] + fre_logger.debug('cmip7 case, extracted brand %s',var_brand) + else: + fre_logger.warning('cmip7 case, extracted multiple brands %s, attempting disambiguation', + brands) + var_brand = filter_brands( + brands, target_var, mip_var_cfgs, + has_time_bnds = 'time_bnds' in ds.variables, + input_vert_dim = get_vertical_dimension(ds, target_var) + ) + else: + fre_logger.error('cmip7 case detected, but dimensions of input data do not match ' + 'any of those found for the associated brands.') + raise ValueError + else: + fre_logger.debug('non-cmip7 case detected, skipping variable brands') + + # try to read what coordinate(s) we're going to be expecting for the variable according to the mip table and compare + expected_mip_coord_dims = None + try: + if exp_cfg_mip_era == 'CMIP7': + expected_mip_coord_dims = mip_var_cfgs["variable_entry"][f'{target_var}_{var_brand}']["dimensions"] + else: + expected_mip_coord_dims = mip_var_cfgs["variable_entry"][target_var]["dimensions"] + + fre_logger.info( + 'I am hoping to find data for the following coordinate dimensions:\n' + ' expected_mip_coord_dims = %s\n', + expected_mip_coord_dims + ) + except Exception as exc: #uncovered + fre_logger.warning( + 'could not get expected coordinate dimensions for %s. ' + ' in mip_var_cfgs file %s. \n exc = %s', + target_var, json_table_config, exc + ) + + # Attempt to read lat/lon coordinates and bnds. will check for none later + fre_logger.info('attempting to read coordinate, lat') + lat = from_dis_gimme_dis(from_dis=ds, gimme_dis="lat") + fre_logger.info('attempting to read coordinate BNDS, lat_bnds') + lat_bnds = from_dis_gimme_dis(from_dis=ds, gimme_dis="lat_bnds") + fre_logger.info('attempting to read coordinate, lon') + lon = from_dis_gimme_dis(from_dis=ds, gimme_dis="lon") + fre_logger.info('attempting to read coordinate BNDS, lon_bnds') + lon_bnds = from_dis_gimme_dis(from_dis=ds, gimme_dis="lon_bnds") + + # read in time_coords + units + fre_logger.info('attempting to read coordinate time, and units...') + time_coords = from_dis_gimme_dis(from_dis=ds, gimme_dis='time') + time_coord_units = ds["time"].units + fre_logger.info(" time_coord_units = %s", time_coord_units) + + # check the calendar of the input netcdf file time coordinate, if present + time_coords_calendar=None + try: # first attempt + time_coords_calendar = ds['time'].calendar.lower() + except: + fre_logger.debug("could not find calendar attribute on time axis. moving on.") + + if time_coords_calendar is None: + try: # second attempt if first didn't work + time_coords_calendar=ds['time'].calendar_type.lower() + except: + fre_logger.debug("could not find calendar_type attribute on time axis. moving on.") + + # if it's still None, give a warning and move on. + if time_coords_calendar is None: + fre_logger.warning("WARNING input netcdf file's time coordinates do not have a calendar nor calendar_type field" + "this output could have the wrong calendar!") + else: + with open(json_exp_config, "r", encoding="utf-8") as file: + exp_cfg_calendar = json.load(file)['calendar'] + if exp_cfg_calendar != time_coords_calendar: + raise ValueError(f"data calendar type {time_coords_calendar} " + f"does not match input config calendar type: {exp_cfg_calendar}") + + # read in time_bnds, if present + fre_logger.info('attempting to read coordinate BNDS, time_bnds') + time_bnds = from_dis_gimme_dis(from_dis=ds, gimme_dis='time_bnds') + + # determine the vertical dimension by looping over netcdf variables + vert_dim = get_vertical_dimension(ds, target_var) # returns int(0) if not present + fre_logger.info("Vertical dimension of %s: %s", target_var, vert_dim) + + # Check var_dim and vert_dim and assign lev if relevant. + lev, lev_units = None, "1" + lev_bnds = None + if vert_dim != 0: + if vert_dim.lower() not in ACCEPTED_VERT_DIMS: + raise ValueError(f'var_dim={var_dim}, vert_dim = {vert_dim} is not supported') #uncovered + lev = ds[vert_dim] + if vert_dim.lower() != "landuse": + lev_units = ds[vert_dim].units + + process_tripolar_data = all([uses_ocean_grid, lat is None, lon is None]) + statics_file_path = None + xh, yh = None, None + xh_dim, yh_dim = None, None + xh_bnds, yh_bnds = None, None + vertex = None + if process_tripolar_data: + try: + fre_logger.info('netcdf_file is %s', netcdf_file) + + # first, try the gold-standard archived ocean statics file + statics_file_path = find_gold_ocean_statics_file( + put_copy_here=f'/net2/{getpass.getuser()}') + + # fall back to the legacy FRE-bronx directory convention + if statics_file_path is None: + fre_logger.info('gold statics not available, falling back to find_statics_file') + statics_file_path = find_statics_file(prev_path) + + fre_logger.info('statics_file_path is %s', statics_file_path) + except Exception as exc: #uncovered + fre_logger.warning( + f'exc = {exc}\n' + 'an ocean statics file is needed, but it could not be found.\n' + ' moving on and doing my best, but I am probably going to break' + ) + raise FileNotFoundError('statics file not found.') from exc + + + fre_logger.info("statics file found.") + + statics_file_name = Path(statics_file_path).name + put_statics_file_here = str(Path(netcdf_file).parent) + shutil.copy(statics_file_path, put_statics_file_here) + del statics_file_path + + statics_file_path = put_statics_file_here + '/' + statics_file_name + fre_logger.info('statics file path is now: %s', statics_file_path) + + # statics file read + statics_ds = nc.Dataset(statics_file_path, 'r') + + # grab the lat/lon points, have shape (yh, xh) + fre_logger.info('reading geolat and geolon coordinates of cell centers from statics file') + statics_lat = from_dis_gimme_dis(statics_ds, 'geolat') + statics_lon = from_dis_gimme_dis(statics_ds, 'geolon') + + fre_logger.info('') + print_data_minmax(statics_lat, "statics_lat") + print_data_minmax(statics_lon, "statics_lon") + fre_logger.info('') + + # spherical lat and lon coords + fre_logger.info('creating lat and lon variables in temp file') + lat = ds.createVariable('lat', statics_lat.dtype, ('yh', 'xh')) + lon = ds.createVariable('lon', statics_lon.dtype, ('yh', 'xh')) + lat[:] = statics_lat[:] + lon[:] = statics_lon[:] + + fre_logger.info('') + print_data_minmax(lat[:], "lat") + print_data_minmax(lon[:], "lon") + fre_logger.info('') + + # grab the corners of the cells, should have shape (yh+1, xh+1) + fre_logger.info('reading geolat and geolon coordinates of cell corners from statics file') + lat_c = from_dis_gimme_dis(statics_ds, 'geolat_c') + lon_c = from_dis_gimme_dis(statics_ds, 'geolon_c') + + fre_logger.info('') + print_data_minmax(lat_c, "lat_c") + print_data_minmax(lon_c, "lon_c") + fre_logger.info('') + + # vertex + fre_logger.info('creating vertex dimension') + vertex = 4 + ds.createDimension('vertex', vertex) + + # lat and lon bnds + fre_logger.info('creating lat and lon bnds from geolat and geolon of corners') + lat_bnds = ds.createVariable('lat_bnds', lat_c.dtype, ('yh', 'xh', 'vertex')) + lat_bnds[:, :, 0] = lat_c[1:, 1:] # NE corner + lat_bnds[:, :, 1] = lat_c[1:, :-1] # NW corner + lat_bnds[:, :, 2] = lat_c[:-1, :-1] # SW corner + lat_bnds[:, :, 3] = lat_c[:-1, 1:] # SE corner + + lon_bnds = ds.createVariable('lon_bnds', lon_c.dtype, ('yh', 'xh', 'vertex')) + lon_bnds[:, :, 0] = lon_c[1:, 1:] # NE corner + lon_bnds[:, :, 1] = lon_c[1:, :-1] # NW corner + lon_bnds[:, :, 2] = lon_c[:-1, :-1] # SW corner + lon_bnds[:, :, 3] = lon_c[:-1, 1:] # SE corner + + fre_logger.info('') + print_data_minmax(lat_bnds[:], "lat_bnds") + print_data_minmax(lon_bnds[:], "lon_bnds") + fre_logger.info('') + + # grab the h-point lat and lon + fre_logger.info('reading yh, xh') + yh = from_dis_gimme_dis(ds, 'yh') + xh = from_dis_gimme_dis(ds, 'xh') + + fre_logger.info('') + print_data_minmax(yh[:], "yh") + print_data_minmax(xh[:], "xh") + fre_logger.info('') + + yh_dim = len(yh) + xh_dim = len(xh) + + # read the q-point native-grid lat lon points + fre_logger.info('reading yq, xq from statics file') + yq = from_dis_gimme_dis(statics_ds, 'yq') + xq = from_dis_gimme_dis(statics_ds, 'xq') + + fre_logger.info('') + print_data_minmax(yq, "yq") + print_data_minmax(xq, "xq") + fre_logger.info('') + + xq_dim = len(xq) + yq_dim = len(yq) + + if any( [yh_dim != (yq_dim - 1), + xh_dim != (xq_dim - 1)]): + raise ValueError( #uncovered + 'the number of h-point lat/lon coordinates is inconsistent with the number of\n' + 'q-point lat/lon coordinates! i.e. ( hpoint_dim != qpoint_dim-1 )\n' + f'yh_dim = {yh_dim}\n' + f'xh_dim = {xh_dim}\n' + f'yq_dim = {yq_dim}\n' + f'xq_dim = {xq_dim}' + ) + + # create h-point bounds from the q-point lat lons + fre_logger.info('creating yh_bnds, xh_bnds from yq, xq') + + yh_bnds = ds.createVariable('yh_bnds', yq.dtype, ('yh', 'nv')) + for i in range(0, yh_dim): + yh_bnds[i, 0] = yq[i] + yh_bnds[i, 1] = yq[i + 1] + + xh_bnds = ds.createVariable('xh_bnds', xq.dtype, ('xh', 'nv')) + for i in range(0, xh_dim): + xh_bnds[i, 0] = xq[i] + xh_bnds[i, 1] = xq[i + 1] + if i % 200 == 0: + fre_logger.info('AFTER assignment: xh_bnds[%d][0] = %s', i, xh_bnds[i][0]) + fre_logger.info('AFTER assignment: xh_bnds[%d][1] = %s', i, xh_bnds[i][1]) + fre_logger.info('type(xh_bnds[%d][1]) = %s', i, type(xh_bnds[i][1])) + + fre_logger.info('') + print_data_minmax(yh_bnds[:], "yh_bnds") + print_data_minmax(xh_bnds[:], "xh_bnds") + fre_logger.info('') + + # now we set up the cmor module object + # initialize CMOR + cmor.setup( + netcdf_file_action=CMOR_NC_FILE_ACTION, + set_verbosity=CMOR_VERBOSITY, + exit_control=CMOR_EXIT_CTL, + create_subdirectories=CMOR_MK_SUBDIRS, + logfile=CMOR_LOG + ) + + # read experiment configuration file + fre_logger.info("cmor is opening: json_exp_config = %s", json_exp_config) + cmor.dataset_json(json_exp_config) + + # load CMOR table + fre_logger.info("cmor is loading+setting json_table_config = %s", json_table_config) + loaded_cmor_table_cfg = cmor.load_table(json_table_config) + cmor.set_table(loaded_cmor_table_cfg) + + # if ocean tripolar grid, we need the CMIP grids configuration file. load it but don't set the table yet. + json_grids_config, loaded_cmor_grids_cfg = None, None + if process_tripolar_data: + json_grids_config = f'{Path(json_table_config).parent}/{exp_cfg_mip_era}_grids.json' + fre_logger.info('cmor is loading/opening %s', json_grids_config) + loaded_cmor_grids_cfg = cmor.load_table(json_grids_config) + cmor.set_table(loaded_cmor_grids_cfg) + + # setup cmor latitude axis if relevant + cmor_y = None + if process_tripolar_data: + fre_logger.warning('calling cmor.axis for a projected y coordinate!!') + cmor_y = cmor.axis("y_deg", coord_vals=yh[:], cell_bounds=yh_bnds[:], units="degrees") + elif lat is None: + fre_logger.warning('lat or lat_bnds is None, skipping assigning cmor_y') + else: + fre_logger.info('assigning cmor_y') + if lat_bnds is None: + cmor_y = cmor.axis("latitude", coord_vals=lat[:], units="degrees_N") #uncovered + else: + cmor_y = cmor.axis("latitude", coord_vals=lat[:], cell_bounds=lat_bnds, units="degrees_N") + fre_logger.info('DONE assigning cmor_y') + + # setup cmor longitude axis if relevant + cmor_x = None + if process_tripolar_data: + fre_logger.warning('calling cmor.axis for a projected x coordinate!!') + cmor_x = cmor.axis("x_deg", coord_vals=xh[:], cell_bounds=xh_bnds[:], units="degrees") + elif lon is None: + fre_logger.warning('lon or lon_bnds is None, skipping assigning cmor_x') + else: + fre_logger.info('assigning cmor_x') + if lon_bnds is None: + cmor_x = cmor.axis("longitude", coord_vals=lon[:], units="degrees_E") #uncovered + else: + cmor_x = cmor.axis("longitude", coord_vals=lon[:], cell_bounds=lon_bnds, units="degrees_E") + fre_logger.info('DONE assigning cmor_x') + + cmor_grid = None + if process_tripolar_data: + fre_logger.warning('setting cmor.grid, process_tripolar_data = %s', process_tripolar_data) + cmor_grid = cmor.grid(axis_ids=[cmor_y, cmor_x], + latitude=lat[:], longitude=lon[:], + latitude_vertices=lat_bnds[:], + longitude_vertices=lon_bnds[:]) + + # now that we are done with setting the grid, we can go back to the usual approach + cmor.set_table(loaded_cmor_table_cfg) + + # setup cmor time axis if relevant + cmor_time = None + ntimes_passed = None + fre_logger.info('assigning cmor_time') + try: + fre_logger.info('assigning cmor_time using time_bnds...') + ntimes_passed=len(time_coords) + fre_logger.debug("Executing: \n" + "cmor.axis('time', \n" + " coord_vals = %s, \n" + " length = %s, \n" + " cell_bounds = %s, units = %s)", + time_coords, ntimes_passed, time_bnds, time_coord_units + ) + cmor_time = cmor.axis("time", + units=time_coord_units, + length=ntimes_passed, + coord_vals=time_coords, + cell_bounds=time_bnds, + interval=None)#interval='mon')# + except Exception as exc: #ValueError as exc: #uncovered + fre_logger.error('exc is %s', str(exc)) + fre_logger.info('assigning cmor_time WITHOUT time_bnds...') + ntimes_passed=len(time_coords) + fre_logger.debug("Executing: \n" + "cmor_time = cmor.axis('time', \n" + " coord_vals = %s, \n" + " length = %s, \n" + " cell_bounds = None, units = %s)", + time_coords, ntimes_passed, time_coord_units + ) + cmor_time = cmor.axis("time", + units=time_coord_units, + length=ntimes_passed, + coord_vals=time_coords, + cell_bounds=None, + interval=None)#interval='mon') + + fre_logger.info('DONE assigning cmor_time') + + # other vertical-axis-relevant initializations + save_ps, ps, ips = False, None, None + ierr_ap, ierr_b = None, None + + # set cmor vertical axis if relevant + cmor_z = None + if lev is not None: + fre_logger.info('assigning cmor_z') + + if vert_dim.lower() in NON_HYBRID_SIGMA_COORDS: + fre_logger.info('non-hybrid sigma coordinate case') + if vert_dim.lower() != "landuse": + cmor_vert_dim_name = vert_dim + cmor_z = cmor.axis(cmor_vert_dim_name, + coord_vals=lev[:], units=lev_units) + else: + landuse_str_list = ['primary_and_secondary_land', 'pastures', 'crops', 'urban'] + cmor_vert_dim_name = "landUse" + cmor_z = cmor.axis(cmor_vert_dim_name, + coord_vals=np.array( + landuse_str_list, + dtype=f'S{len(landuse_str_list[0])}' + ), + units=lev_units) + + elif vert_dim in DEPTH_COORDS: + try: + lev_bnds = create_lev_bnds(bound_these=lev, with_these=ds['z_i']) + fre_logger.info('created lev_bnds...') + except Exception as exc: + fre_logger.error("the cmor module always requires vertical levels to have bounds.") + raise KeyError("CMOR requires the input data have vertical level boundaries (bnds)") from exc + + fre_logger.info('lev_bnds = \n%s', lev_bnds) + cmor_z = cmor.axis('depth_coord', + coord_vals=lev[:], + units=lev_units, + cell_bounds=lev_bnds) + + elif vert_dim in ALT_HYBRID_SIGMA_COORDS: + # find the ps file nearby + ps_file = netcdf_file.replace(f'.{target_var}.nc', '.ps.nc') + ds_ps = nc.Dataset(ps_file) + ps = from_dis_gimme_dis(ds_ps, 'ps') + + # assign lev_half specifics + if vert_dim == "levhalf": + cmor_z = cmor.axis("alternate_hybrid_sigma_half", + coord_vals=lev[:], + units=lev_units) + ierr_ap = cmor.zfactor(zaxis_id=cmor_z, + zfactor_name="ap_half", + axis_ids=[cmor_z, ], + zfactor_values=ds["ap_bnds"][:], + units=ds["ap_bnds"].units) + ierr_b = cmor.zfactor(zaxis_id=cmor_z, + zfactor_name="b_half", + axis_ids=[cmor_z, ], + zfactor_values=ds["b_bnds"][:], + units=ds["b_bnds"].units) + else: + cmor_z = cmor.axis("alternate_hybrid_sigma", + coord_vals=lev[:], + units=lev_units, + cell_bounds=ds[vert_dim + "_bnds"]) + ierr_ap = cmor.zfactor(zaxis_id=cmor_z, + zfactor_name="ap", + axis_ids=[cmor_z, ], + zfactor_values=ds["ap"][:], + zfactor_bounds=ds["ap_bnds"][:], + units=ds["ap"].units) + ierr_b = cmor.zfactor(zaxis_id=cmor_z, + zfactor_name="b", + axis_ids=[cmor_z, ], + zfactor_values=ds["b"][:], + zfactor_bounds=ds["b_bnds"][:], + units=ds["b"].units) + + fre_logger.info('ierr_ap after calling cmor_zfactor: %s\n', ierr_ap) + fre_logger.info('ierr_b after calling cmor_zfactor: %s', ierr_b) + + axis_ids = [] + if cmor_time is not None: + fre_logger.info('appending cmor_time to axis_ids list...') + axis_ids.append(cmor_time) + fre_logger.info('axis_ids now = %s', axis_ids) + # might there need to be a conditional check for tripolar ocean data here as well? TODO + if cmor_y is not None: + fre_logger.info('appending cmor_y to axis_ids list...') + axis_ids.append(cmor_y) + fre_logger.info('axis_ids now = %s', axis_ids) + if cmor_x is not None: + fre_logger.info('appending cmor_x to axis_ids list...') + axis_ids.append(cmor_x) + fre_logger.info('axis_ids now = %s', axis_ids) + + ips = cmor.zfactor(zaxis_id=cmor_z, + zfactor_name="ps", + axis_ids=axis_ids, + units="Pa") + save_ps = True + + + fre_logger.info('DONE assigning cmor_z') + + axes = [] + if cmor_time is not None: + fre_logger.info('appending cmor_time to axes list...') + axes.append(cmor_time) + fre_logger.info('axes now = %s', axes) + + if cmor_z is not None: + fre_logger.info('appending cmor_z to axes list...') + axes.append(cmor_z) + fre_logger.info('axes now = %s', axes) + + if process_tripolar_data: + axes.append(cmor_grid) + else: + if cmor_y is not None: + fre_logger.info('appending cmor_y to axes list...') + axes.append(cmor_y) + fre_logger.info('axes now = %s', axes) + if cmor_x is not None: + fre_logger.info('appending cmor_x to axes list...') + axes.append(cmor_x) + fre_logger.info('axes now = %s', axes) + + # read positive/units attribute and create cmor_var + #units = mip_var_cfgs["variable_entry"][target_var]["units"] + if exp_cfg_mip_era == 'CMIP7': + units = mip_var_cfgs["variable_entry"][f'{target_var}_{var_brand}']["units"] + else: + units = mip_var_cfgs["variable_entry"][target_var]["units"] + fre_logger.info("units = %s", units) + + #positive = mip_var_cfgs["variable_entry"][target_var]["positive"] + if exp_cfg_mip_era == 'CMIP7': + positive = mip_var_cfgs["variable_entry"][f'{target_var}_{var_brand}']["positive"] + else: + positive = mip_var_cfgs["variable_entry"][target_var]["positive"] + fre_logger.info("positive = %s", positive) + + if exp_cfg_mip_era == 'CMIP7': + fre_logger.info('cmor.variable call: for cmip7_target_var = %s ', f'{target_var}_{var_brand}') + cmor_var = cmor.variable(f'{target_var}_{var_brand}', units, axes, + missing_value = var_missing_val, + positive = positive) + fre_logger.info('DONE cmor.variable call: for cmip7_target_var = %s ',f'{target_var}_{var_brand}') + + else: + fre_logger.info('cmor.variable call: for target_var = %s ',target_var) + cmor_var = cmor.variable(target_var, units, axes, + missing_value = var_missing_val, + positive = positive) + fre_logger.info('DONE cmor.variable call: for target_var = %s ',target_var) + + # Write the output to disk + #fre_logger.debug('var is: %s', var) + fre_logger.info("cmor.write call: for var data into cmor_var") + cmor.write(cmor_var, var) + fre_logger.info("DONE cmor.write call: for var data into cmor_var") + if save_ps: + if any([ips is None, ps is None]): + fre_logger.warning('ps or ips is None!, but save_ps is True!\n' #uncovered + 'ps = %s, ips = %s\n' + 'skipping ps writing!', ps, ips) + else: + fre_logger.info("cmor.write call: for interp-pressure data (ips)") + cmor.write(ips, ps, store_with=cmor_var, ntimes_passed=ntimes_passed) + fre_logger.info("DONE cmor.write call: for interp-pressure data (ips)") + + fre_logger.info("cmor.close call: for cmor_var") + filename = cmor.close(cmor_var, file_name=True, preserve=False) + fre_logger.info("DONE cmor.close call: for cmor_var") + filename = str( Path(filename).resolve() ) + fre_logger.info("returned by cmor.close: filename = %s", filename) + fre_logger.info('closing netcdf4 dataset... ds') + ds.close() + fre_logger.info('tearing-down the cmor module instance') + cmor.close() + + fre_logger.info('-------------------------- END rewrite_netcdf_file_var call -----\n\n') + return filename + + +def cmorize_target_var_files(indir: str = None, + target_var: str = None, + local_var: str = None, + iso_datetime_range_arr: List[str] = None, + name_of_set: str = None, + json_exp_config: str = None, + outdir: str = None, + mip_var_cfgs: Dict[str, Any] = None, + json_table_config: str = None, + run_one_mode: bool = False): + """ + CMORize a target variable across all NetCDF files in a directory. + + :param indir: Path to the directory containing NetCDF files to process. + :type indir: str + :param target_var: Name of the variable to process in each file. + :type target_var: str + :param local_var: Local/filename variable name (often identical to target_var). + :type local_var: str + :param iso_datetime_range_arr: List of ISO datetime strings, each identifying a specific file. + :type iso_datetime_range_arr: list of str + :param name_of_set: Post-processing component or label for the targeted files. + :type name_of_set: str + :param json_exp_config: Path to experiment configuration JSON file. + :type json_exp_config: str + :param outdir: Output directory root for CMORized files. + :type outdir: str + :param mip_var_cfgs: Variable table from the MIP table JSON config. + :type mip_var_cfgs: dict + :param json_table_config: Path to MIP table JSON file. + :type json_table_config: str + :param run_one_mode: If True, processes only one file and exits. + :type run_one_mode: bool, optional + :raises ValueError: See function body for details. + :raises OSError: See function body for details. + :raises Exception: See function body for details. + :return: None + :rtype: None + + .. note:: Copies files to a temporary directory, runs CMORization, moves results to output, cleans up temp files. + """ + + fre_logger.info("local_var = %s to be used for file-targeting.\n" + "target_var = %s to be used for reading the data \n" + "from the file\n" + "outdir = %s", local_var, target_var, outdir) + + # determine a tmp dir for working on files. + tmp_dir = create_tmp_dir(outdir, json_exp_config) + '/' + fre_logger.info('will use tmp_dir=%s', tmp_dir) + + # loop over sets of dates, each one pointing to a file + nc_fls = {} + for i, iso_datetime in enumerate(iso_datetime_range_arr): + # why is nc_fls a filled list/array/object thingy here? see above line + nc_fls[i] = f"{indir}/{name_of_set}.{iso_datetime}.{local_var}.nc" + + fre_logger.info("input file = %s", nc_fls[i]) + if not Path(nc_fls[i]).exists(): + fre_logger.warning("input file(s) not found. Moving on.") #uncovered + continue + + if not Path(nc_fls[i]).is_absolute(): + nc_fls[i]=str(Path(nc_fls[i]).resolve()) + + # create a copy of the input file with local var name into the work directory + nc_file_work = f"{tmp_dir}{name_of_set}.{iso_datetime}.{local_var}.nc" + + fre_logger.info("nc_file_work = %s", nc_file_work) + shutil.copy(nc_fls[i], nc_file_work) + + # if the ps file exists, we'll copy it to the work directory too + nc_ps_file = nc_fls[i].replace(f'.{local_var}.nc', '.ps.nc') + nc_ps_file_work = nc_file_work.replace(f'.{local_var}.nc', '.ps.nc') + if Path(nc_ps_file).exists(): + fre_logger.info("nc_ps_file_work = %s", nc_ps_file_work) + shutil.copy(nc_ps_file, nc_ps_file_work) + + # TODO think of better way to write this kind of conditional data movement... + # now we have a file in our targets, point CMOR to the configs and the input file(s) + make_cmor_write_here = tmp_dir + # make sure we know where we are writing, or else! + if not Path(make_cmor_write_here).exists(): + raise ValueError(f'\ntmp_dir = \n{tmp_dir}\ncannot be found/created/resolved!') #uncovered + + gotta_go_back_here = os.getcwd() + try: + fre_logger.warning("changing directory to: \n%s", make_cmor_write_here) + os.chdir(make_cmor_write_here) + except Exception as exc: #uncovered + raise OSError(f'(cmorize_target_var_files) could not chdir to {make_cmor_write_here}') from exc + + fre_logger.info("calling rewrite_netcdf_file_var") + try: + local_file_name = rewrite_netcdf_file_var(mip_var_cfgs, + local_var, + nc_file_work, + target_var, + json_exp_config, + json_table_config, + prev_path=nc_fls[i] ) + except Exception as exc: #uncovered + raise Exception( + 'problem with rewrite_netcdf_file_var. ' + f'exc={exc}\n' + 'exiting and executing finally block.') from exc + finally: # should always execute, errors or not! + fre_logger.warning('finally, changing directory to: \n%s', gotta_go_back_here) + os.chdir(gotta_go_back_here) + +# assert False, "made it to break-point for current work, good job" + + # now that CMOR has rewritten things... we can take our post-rewriting actions + # first, remove /CMOR_tmp/ from the output path. + if not Path(local_file_name).is_absolute(): + raise ValueError(f'local_file_name should be an absolute path, not a relative one. \n ' + f'local_file_name = {local_file_name}') + + fre_logger.info('local_file_name = %s', local_file_name) + filename = local_file_name.replace('/CMOR_tmp/','/') + fre_logger.info("filename = %s", filename) + + # the final output file directory will be... + filedir = Path(filename).parent + fre_logger.info("FINAL OUTPUT FILE DIR WILL BE filedir = %s", filedir) + try: + fre_logger.info('ATTEMPTING TO CREATE filedir=%s', filedir) + os.makedirs(filedir) + except FileExistsError: + fre_logger.warning('directory %s already exists!', filedir) + + mv_cmd = f"mv {local_file_name} {filedir}" + fre_logger.info("moving files...\n%s", mv_cmd) + subprocess.run(mv_cmd, shell=True, check=True) + + # ------ refactor this into function? #TODO + # ------ what is the use case for this logic really?? + filename_no_nc = filename[:filename.rfind(".nc")] + chunk_str = filename_no_nc[-6:] + if not chunk_str.isdigit(): + fre_logger.warning('chunk_str is not a digit: chunk_str = %s', chunk_str) #uncovered + filename_corr = f"{filename[:filename.rfind('.nc')]}_{iso_datetime}.nc" + mv_cmd = f"mv {filename} {filename_corr}" + fre_logger.warning("moving files, strange chunkstr logic...\n%s", mv_cmd) + subprocess.run(mv_cmd, shell=True, check=True) + # ------ end refactor this into function? + + # delete files in work dirs + if Path(nc_file_work).exists(): + Path(nc_file_work).unlink() + + if Path(nc_ps_file_work).exists(): + Path(nc_ps_file_work).unlink() + + if run_one_mode: + fre_logger.warning('run_one_mode is True!!!!') + fre_logger.warning('done processing one file!!!') + break + + +def cmorize_all_variables_in_dir(vars_to_run: Dict[str, Any], + indir: str, + iso_datetime_range_arr: List[str], + name_of_set: str, + json_exp_config: str, + outdir: str, + mip_var_cfgs: Dict[str, Any], + json_table_config: str, + run_one_mode: bool) -> int: + """ + CMORize all variables in a directory according to a variable mapping. + + :param vars_to_run: Mapping of local variable names (in filenames) to target variable names (in NetCDF). + :type vars_to_run: dict + :param indir: Directory containing NetCDF files to process. + :type indir: str + :param iso_datetime_range_arr: List of ISO datetime strings to identify files. + :type iso_datetime_range_arr: list of str + :param name_of_set: Post-processing component or set label. + :type name_of_set: str + :param json_exp_config: Path to experiment configuration JSON file. + :type json_exp_config: str + :param outdir: Output directory root for CMORized files. + :type outdir: str + :param mip_var_cfgs: Variable table from the MIP table JSON config. + :type mip_var_cfgs: dict + :param json_table_config: Path to MIP table JSON file. + :type json_table_config: str + :param run_one_mode: If True, process only one file per variable. + :type run_one_mode: bool + :return: 0 if last file processed was successful, 1 if last file processed failed, -1 if no files were processed. + :rtype: int + + .. note:: Errors for individual variables are logged and processing continues (except for run_one_mode). + """ + + # loop over local-variable:target-variable pairs in vars_to_run + return_status = -1 + for local_var in vars_to_run: + # if the target-variable is "good", get the name of the data inside the netcdf file. + target_var = vars_to_run[local_var] # often equiv to local_var but not necessarily. + if local_var != target_var: + fre_logger.warning('local_var == %s != %s == target_var\n' + 'i am expecting %s to be in the filename, and i expect the variable\n' + 'in that file to be named %s', local_var, target_var, local_var, target_var) + + fre_logger.info('........beginning CMORization for %s/%s..........', local_var, target_var) + try: + cmorize_target_var_files(indir, target_var, local_var, iso_datetime_range_arr, + name_of_set, json_exp_config, outdir, + mip_var_cfgs, json_table_config, run_one_mode) + return_status = 0 + except Exception as exc: #uncovered + return_status = 1 + fre_logger.warning('!!!EXCEPTION CAUGHT!!! !!!READ THE NEXT LINE!!!') + fre_logger.warning('exc=%s', exc) + fre_logger.warning('this message came from within cmorize_target_var_files') + fre_logger.warning('COULD NOT PROCESS: %s/%s...moving on', local_var, target_var) + # log an omitted variable here... + + if run_one_mode: + fre_logger.warning('run_one_mode is True. breaking vars_to_run loop') + break + return return_status + + +def cmor_run_subtool(indir: str = None, + json_var_list: str = None, + json_table_config: str = None, + json_exp_config: str = None, + outdir: str = None, + run_one_mode: Optional[bool] = False, + opt_var_name: Optional[str] = None, + grid: Optional[str] = None, + grid_label: Optional[str] = None, + nom_res: Optional[str] = None, + start: Optional[str] = None, + stop: Optional[str] = None, + calendar_type: Optional[str] = None) -> int: + """ + Main entry point for CMORization workflow, steering all routines in this file. + + :param indir: Directory containing NetCDF files to process. + :type indir: str + :param json_var_list: Path to JSON file with variable mapping (local to target names). + :type json_var_list: str + :param json_table_config: Path to MIP table JSON file (per-variable metadata). + :type json_table_config: str + :param json_exp_config: Path to experiment configuration JSON file (for header metadata). + :type json_exp_config: str + :param outdir: Output directory root for CMORized files. + :type outdir: str + :param run_one_mode: If True, process only one file per variable. + :type run_one_mode: bool, optional + :param opt_var_name: If provided, only process this variable. + :type opt_var_name: str, optional + :param grid: Grid description (if gridding is specified). + :type grid: str, optional + :param grid_label: Grid label (must match controlled vocabulary if provided). + :type grid_label: str, optional + :param nom_res: Nominal resolution for grid (must match controlled vocabulary if provided). + :type nom_res: str, optional + :param start: Start year (YYYY) for files to process. + :type start: str, optional + :param stop: Stop year (YYYY) for files to process. + :type stop: str, optional + :param calendar_type: CF-compliant calendar type. + :type calendar_type: str, optional + :raises ValueError: If required parameters are missing or inconsistent. + :raises FileNotFoundError: If required files do not exist. + :return: 0 if successful. + :rtype: int + + .. note:: Updates grid, label, and calendar fields in experiment config if needed. + .. note:: Loads variable mapping and MIP table, filters variables, and orchestrates file processing. + """ + # CHECK req'd inputs + if None in [indir, json_var_list, json_table_config, json_exp_config, outdir]: + raise ValueError('the following input arguments are required:\n' + '[indir, json_var_list, json_table_config, json_exp_config, outdir] = \n' + '[%s, %s, %s, %s, %s]', indir, json_var_list, json_table_config, json_exp_config, outdir) + + # CHECK existence of the exp-specific metadata file + if Path(json_exp_config).exists(): + json_exp_config = str(Path(json_exp_config).resolve()) + else: + raise FileNotFoundError('ERROR: json_exp_config file cannot be opened.\n' + 'json_exp_config = %s', json_exp_config) + + # CHECK mip_era entry of exp config exists, needed ? + try: + exp_cfg_mip_era = get_json_file_data(json_exp_config)['mip_era'].upper() + except KeyError as exc: + raise KeyError('no mip_era entry in experimental metadata configuration, the file is noncompliant!') from exc + + fre_logger.debug('exp_cfg_mip_era = %s', exp_cfg_mip_era) + if exp_cfg_mip_era not in ['CMIP6', 'CMIP7']: + raise ValueError('cmor_mixer only supports CMIP6 and CMIP7 cases') + + if exp_cfg_mip_era == 'CMIP7': + fre_logger.warning('CMIP7 configuration detected, will be expecting and enforcing variable brands.') + + # CHECK optional grid/grid_label/nom_res inputs from exp config, the function raises the potential error conditions + if any( [ grid_label is not None, + grid is not None, + nom_res is not None ] ): + update_grid_and_label(json_exp_config, + grid_label, grid, nom_res, + output_file_path = None) + + #update_outpath(json_exp_config, outpath = outdir, output_file_path = None) + + # CHECK optional grid/grid_label inputs, the function checks the potential error conditions RE CF compliance. + if calendar_type is not None: + update_calendar_type(json_exp_config, calendar_type, output_file_path = None) + + + # open CMOR table config file - need it here for checking the TABLE's variable list + json_table_config = str(Path(json_table_config).resolve()) + fre_logger.info('loading json_table_config = \n%s', json_table_config) + + mip_var_cfgs = get_json_file_data(json_table_config) + mip_fullvar_list = mip_var_cfgs["variable_entry"].keys() + fre_logger.debug('the following variables were read from the table: %s', mip_fullvar_list) + + # make the TABLE's variable list, and brand list (if CMIP7) + mip_var_list, mip_var_brand_list = None, None + if exp_cfg_mip_era == 'CMIP7': + fre_logger.warning('cmip7 capabilities in-development now. extracting brands from variables ' + 'within MIP cmor table configs') + mip_var_list = [ var.split('_')[0] for var in mip_fullvar_list ] + mip_var_brand_list = [ var.split('_')[1] for var in mip_fullvar_list ] + if len(mip_var_list) != len(mip_var_brand_list): + raise ValueError('the number of brands is not one-to-one with the number of variables. check config.') + elif exp_cfg_mip_era == "CMIP6": + mip_var_list = mip_fullvar_list + + fre_logger.debug('list of table variables we will process = \n %s', mip_var_list) + if mip_var_brand_list is not None: + fre_logger.debug('the following brands were extracted from the variables: %s', mip_var_brand_list) + + # open USER input variable list, no brands required regardless of CMIP6/7 + # these are largely for targeting GFDL's input files and reading them + json_var_list = str(Path(json_var_list).resolve()) + fre_logger.debug('loading json_var_list = \n%s', json_var_list) + + var_list = get_json_file_data(json_var_list) + fre_logger.debug('var_list is = \n %s', var_list) + + # CHECK that the user's input variables make sense against those in the targeted table + # if the check(s) pass, the final list of variables to run is stored in vars_to_run + # if opt_var_name is specified, the routinue is short-circuited to care only about opt_var_name + vars_to_run = {} + for local_var in var_list: + if opt_var_name is not None and opt_var_name in mip_var_list: + vars_to_run[opt_var_name] = opt_var_name + break + if var_list[local_var] not in mip_var_list: #mip_var_cfgs["variable_entry"]: + fre_logger.warning('skipping local_var = %s /\n' + 'target_var = %s\n' + 'target_var not found in CMOR variable group', local_var, var_list[local_var]) + continue + + fre_logger.info('%s found in %s', var_list[local_var], Path(json_table_config).name) + vars_to_run[local_var] = var_list[local_var] + fre_logger.info('vars_to_run = %s', vars_to_run) + + # CHECK that there's at least one variable to run after comparing use inputs vars to MIP config input vars + if len(vars_to_run) < 1: + raise ValueError('runnable variable list is of length 0 ' + 'this means no variables in input variable list are in ' + 'the mip table configuration, so there\'s nothing to process!') + if all([opt_var_name is not None, opt_var_name not in list(vars_to_run.keys())]): + raise ValueError('opt_var_name is not None! (== %s)' + '... but the variable is not contained in the target mip table' + '... there\'s nothing to process, exit', opt_var_name) + + fre_logger.info('runnable variable list formed, it is vars_to_run=\n%s', vars_to_run) + + # make list of target files within targeted indir here + # examine input directory to obtain a list of input file targets + fre_logger.info('indir = %s', indir) + indir_filenames = glob.glob(f'{indir}/*.nc') + indir_filenames.sort() + if len(indir_filenames) == 0: + raise ValueError('no files in input target directory = indir = \n%s', indir) + fre_logger.debug('found %s filenames', len(indir_filenames)) + + # name_of_set == component label + name_of_set = Path(indir_filenames[0]).name.split(".")[0] + fre_logger.info('setting name_of_set = %s', name_of_set) + + # make list of iso-datetimes here + iso_datetime_range_arr = [] + get_iso_datetime_ranges(indir_filenames, iso_datetime_range_arr, start, stop) + fre_logger.info('\nfound iso datetimes = %s', iso_datetime_range_arr) + + # no longer needed. + del indir_filenames + + # now we descend into more CPU-heavy work here + return cmorize_all_variables_in_dir( vars_to_run, + indir, iso_datetime_range_arr, name_of_set, json_exp_config, + outdir, mip_var_cfgs, json_table_config, run_one_mode ) diff --git a/fremorizer/cmor_yamler.py b/fremorizer/cmor_yamler.py new file mode 100644 index 0000000..9ba1054 --- /dev/null +++ b/fremorizer/cmor_yamler.py @@ -0,0 +1,290 @@ +""" +YAML-Driven CMORization Workflow Tools +====================================== + +This module powers the ``fremor yaml`` command, steering the CMORization workflow by parsing model-YAML +files that describe target experiments and their configurations. It combines model-level and experiment-level +configuration, parses required metadata and paths, and orchestrates calls to ``cmor_run_subtool`` for each +target variable/component. + +Functions +--------- +- ``cmor_yaml_subtool(...)`` + +.. note:: "yamler" is a portmanteau of "yaml" and "reader". +""" + +from pathlib import Path +import pprint +import logging +import os +from typing import Optional + +try: + from fre.yamltools.combine_yamls_script import consolidate_yamls +except ImportError: + consolidate_yamls = None +from .cmor_mixer import cmor_run_subtool +from .cmor_helpers import ( check_path_existence, iso_to_bronx_chunk, #conv_mip_to_bronx_freq, + get_bronx_freq_from_mip_table ) + +fre_logger = logging.getLogger(__name__) + +def cmor_yaml_subtool( yamlfile: str = None, + exp_name: str = None, + platform: str = None, + target: str = None, + output: Optional[str] = None, + opt_var_name: Optional[str] = None, + run_one_mode: bool = False, + dry_run_mode: bool = False, + start: Optional[str] = None, + stop: Optional[str] = None, + calendar_type: Optional[str] = None, + print_cli_call: bool = True): + """ + Main driver for CMORization using model YAML configuration files. + This routine parses the model YAML, combines configuration, resolves and checks all required + paths and metadata, and orchestrates calls to cmor_run_subtool for each table/component/variable + defined in the configuration. + + :param yamlfile: Path to a model-yaml file holding experiment and workflow configuration. + :type yamlfile: str + :param exp_name: Experiment name (must be present in the YAML file). + :type exp_name: str + :param platform: Platform target (e.g., 'ncrc4.intel'). + :type platform: str + :param target: Compilation target (e.g., 'prod-openmp'). + :type target: str + :param output: filename for YAML output. + :type output: str, optional + :param opt_var_name: If specified, process only files matching this variable name. + :type opt_var_name: str, optional + :param run_one_mode: If True, process only one file and exit. + :type run_one_mode: bool + :param dry_run_mode: If True, print configuration and actions without executing cmor_run_subtool. + :type dry_run_mode: bool + :param start: Four-digit year (YYYY) indicating start of date range to process. + :type start: str, optional + :param stop: Four-digit year (YYYY) indicating end of date range to process. + :type stop: str, optional + :param calendar_type: CF-compliant calendar type. + :type calendar_type: str, optional + :param print_cli_call: When True and dry_run_mode is enabled, print + the equivalent ``fre cmor run`` CLI invocation; when False, print + the Python ``cmor_run_subtool(...)`` call instead. + :type print_cli_call: bool + :raises FileNotFoundError: If required paths do not exist. + :raises OSError: If output directories cannot be created. + :raises ValueError: If required configuration is missing or inconsistent. + :return: None + :rtype: None + + .. note:: Reads and combines YAML and JSON configuration. + .. note:: Performs path, frequency, and gridding checks. + .. note:: Delegates actual CMORization to cmor_run_subtool, except in dry-run mode. + .. note:: All actions and key decisions are logged. + """ + check_path_existence(yamlfile) + + # --------------------------------------------------- + # parsing the target model yaml --------------------- + # --------------------------------------------------- + fre_logger.info('calling consolidate yamls to create a combined cmor-yaml dictionary') + if consolidate_yamls is None: + raise ImportError( + "the 'fremor yaml' command requires fre-cli's yamltools module.\n" + "install it with: pip install fre-cli") + cmor_yaml_dict = consolidate_yamls(yamlfile=yamlfile, + experiment=exp_name, platform=platform, target=target, + use="cmor", output=output)['cmor'] + fre_logger.debug('consolidate_yamls produced the following dictionary of cmor-settings from yamls: \n%s', + pprint.pformat(cmor_yaml_dict) ) + + mip_era = cmor_yaml_dict['mip_era'].upper() + fre_logger.info('mip_era = %s', mip_era) + + # --------------------------------------------------- + # between-logic to form args ---------------------- + # --------------------------------------------------- + + # target input pp directory + pp_dir = os.path.expandvars( + cmor_yaml_dict['directories']['pp_dir'] ) + fre_logger.info('pp_dir = %s', pp_dir) + check_path_existence(pp_dir) + + # directory holding mip table config inputs + cmip_cmor_table_dir = os.path.expandvars( + cmor_yaml_dict['directories']['table_dir'] ) + fre_logger.info('cmip_cmor_table_dir = %s', cmip_cmor_table_dir) + check_path_existence(cmip_cmor_table_dir) + + # final directory housing whole CMOR dir structure at the end of it all + cmorized_outdir = os.path.expandvars( + cmor_yaml_dict['directories']['outdir'] ) + fre_logger.info('cmorized_outdir = %s', cmorized_outdir) + if not Path(cmorized_outdir).exists(): + try: + fre_logger.info('cmorized_outdir does not exist.') + fre_logger.info('attempt to create it...') + Path(cmorized_outdir).mkdir(exist_ok=False, parents=True) + except Exception as exc: #uncovered + raise OSError( + f'could not create cmorized_outdir = {cmorized_outdir} for some reason!') from exc + + # path to input user/experiment configuration, expected by CMOR + json_exp_config = os.path.expandvars( + cmor_yaml_dict['exp_json'] ) + fre_logger.info('json_exp_config = %s', json_exp_config) + check_path_existence(json_exp_config) + + # check start/stop years range to target desired input files + if start is None: + try: + yaml_start = cmor_yaml_dict['start'] + start = yaml_start + except KeyError: + fre_logger.warning( + 'no start year for fre.cmor given anywhere, will start with earliest datetime found in filenames!') + + if stop is None: + try: + yaml_stop = cmor_yaml_dict['stop'] + stop = yaml_stop + except KeyError: + fre_logger.warning( + 'no stop year for fre.cmor given anywhere, will end with latest datetime found in filenames!') + if calendar_type is None: + try: + yaml_calendar_type = cmor_yaml_dict['calendar_type'] + calendar_type = yaml_calendar_type + except KeyError: + fre_logger.warning( + 'no calendar_type for fre.cmor given anywhere, will use what is in %s', json_exp_config) + + # --------------------------------------------------- + # showtime ------------------------------------------ + # --------------------------------------------------- + for cmor_yaml_table_target in cmor_yaml_dict['table_targets']: + table_name = cmor_yaml_table_target['table_name'] + fre_logger.info('table_name = %s', table_name) + + json_mip_table_config = f'{cmip_cmor_table_dir}/{mip_era}_{table_name}.json' + fre_logger.info('json_mip_table_config = %s', json_mip_table_config) + check_path_existence(json_mip_table_config) + + # frequency of data ---- the reason this spot looks kind of awkward is because of the case where + # the table if e.g. Ofx and thus the table's frequency field is smth like 'fx' + # if that's the case, we only demand that the freq field is filled out in the yaml + # which is really more about path resolving than anything. + + # check frequency info from user + freq = cmor_yaml_table_target['freq'] + + # if freq not supplied, behavior depends on MIP era + if freq is None: + if mip_era == 'CMIP7': + # CMIP7 tables do not carry frequency — the user must always specify it + raise ValueError( + f'freq is required for CMIP7 but was not specified for table target {table_name}.\n' + f' CMIP7 MIP tables do not contain frequency metadata.\n' + f' please set freq explicitly (e.g. "monthly", "daily") in the cmor yaml.') + # CMIP6 tables carry frequency — attempt to derive it + fre_logger.info('freq not specified in cmor yaml for table %s, ' + 'attempting to derive from CMIP6 MIP table', table_name) + try: + freq = get_bronx_freq_from_mip_table(json_mip_table_config) + except (KeyError, TypeError): + freq = None + if freq is None: + raise ValueError( + f'not enough frequency information to process variables for {table_name}.\n' + f' freq was not specified in the cmor yaml, and could not be derived from the MIP table.\n' + f' please set freq explicitly (e.g. "monthly", "daily") in the cmor yaml.') + fre_logger.info('derived freq = %s from MIP table %s', freq, json_mip_table_config) + + # update the table target dict so downstream code sees the resolved freq + cmor_yaml_table_target['freq'] = freq + fre_logger.info('freq = %s', freq) + + # gridding info of data ---- revisit/TODO + gridding_dict = cmor_yaml_table_target['gridding'] + fre_logger.debug('gridding_dict = %s', gridding_dict) + grid_label, grid_desc, nom_res = None, None, None + if gridding_dict is not None: + grid_label = gridding_dict['grid_label'] + grid_desc = gridding_dict['grid_desc'] + nom_res = gridding_dict['nom_res'] + if None in [grid_label, grid_desc, nom_res]: + raise ValueError('gridding dictionary, if present, must have all three fields be non-empty.') + # gridding info of data ---- revisit + + table_components_list = cmor_yaml_table_target['target_components'] + for targ_comp_config in table_components_list: + component = targ_comp_config['component_name'] + bronx_chunk = iso_to_bronx_chunk(targ_comp_config['chunk']) + data_series_type = targ_comp_config['data_series_type'] + + json_var_list = os.path.expandvars( + targ_comp_config['variable_list'] + ) + fre_logger.info('json_var_list = %s', json_var_list) + indir = f'{pp_dir}/{component}/{data_series_type}/{freq}/{bronx_chunk}' + fre_logger.info('indir = %s', indir) + + fre_logger.info('PROCESSING: ( %s, %s )', table_name, component) + cmor_run_call_outdir=f'{cmorized_outdir}/{component}/{table_name}' + + + if dry_run_mode: + if print_cli_call: + fre_logger.info( '--DRY RUN CLI CALL---\n' + 'fre -v -v cmor run \\ \n' + f' --indir {indir} \\ \n' + f' --varlist {json_var_list} \\ \n' + f' --table_config {json_mip_table_config} \\ \n' + f' --exp_config {json_exp_config} \\ \n' + f' --outdir {cmor_run_call_outdir} \\ \n' + f' --run_one \\ \n' + f' --opt_var_name {opt_var_name} ,\n' + f' --grid_desc "{grid_desc}" \\ \n' + f' --grid_label {grid_label} \\ \n' + f' --nom_res "{nom_res}" \\ \n' + f' --start {start} \\ \n' + f' --stop {stop} \\ \n' + f' --calendar {calendar_type}' + '\n' ) + else: + fre_logger.info( '--DRY RUN CALL---\n' + 'cmor_run_subtool(\n' + f' indir = {indir} ,\n' + f' json_var_list = {json_var_list} ,\n' + f' json_table_config = {json_mip_table_config} ,\n' + f' json_exp_config = {json_exp_config} ,\n' + f' outdir = {cmor_run_call_outdir} ,\n' + f' run_one_mode = {run_one_mode} ,\n' + f' opt_var_name = {opt_var_name} ,\n' + f' grid = {grid_desc} ,\n' + f' grid_label = {grid_label} ,\n' + f' nom_res = {nom_res} ,\n' + f' start = {start} ,\n' + f' stop = {stop} ,\n' + f' calendar_type = {calendar_type}' + ')\n' ) + continue + cmor_run_subtool( #uncovered + indir = indir , + json_var_list = json_var_list , + json_table_config = json_mip_table_config , + json_exp_config = json_exp_config , + outdir = cmor_run_call_outdir , + run_one_mode = run_one_mode , + opt_var_name = opt_var_name , + grid = grid_desc , + grid_label = grid_label , + nom_res = nom_res , + start = start , + stop = stop , + calendar_type = calendar_type + ) diff --git a/fremorizer/fremor.py b/fremorizer/fremor.py new file mode 100644 index 0000000..b9b9457 --- /dev/null +++ b/fremorizer/fremor.py @@ -0,0 +1,292 @@ +''' fremorizer CLI entry point: fremor ''' + +import logging + +import click + +from . import __version__ as version, FORMAT +from .cmor_finder import cmor_find_subtool, make_simple_varlist +from .cmor_mixer import cmor_run_subtool +from .cmor_yamler import cmor_yaml_subtool +from .cmor_config import cmor_config_subtool + +fre_logger = logging.getLogger(__name__) + +OPT_VAR_NAME_HELP="optional, specify a variable name to specifically process only filenames " + \ + "matching that variable name. I.e., this string help target local_vars, not " + \ + "target_vars." +VARLIST_HELP="path pointing to a json file containing directory of key/value pairs. " + \ + "the keys are the \'local\' names used in the filename, and the values " + \ + "pointed to by those keys are strings representing the name of the variable " + \ + "contained in targeted files. the key and value are often the same, " + \ + "but it is not required." +RUN_ONE_HELP="process only one file, then exit. mostly for debugging and isolating issues." +DRY_RUN_HELP="don't call the cmor_mixer subtool, just printout what would be called and move on until natural exit" +START_YEAR_HELP = 'string representing the minimum calendar year CMOR should start processing for. ' + \ + 'currently, only YYYY format is supported.' +STOP_YEAR_HELP = 'string representing the maximum calendar year CMOR should stop processing for. ' + \ + 'currently, only YYYY format is supported.' + + +@click.version_option( + package_name = "fremorizer", + version = version +) +@click.group( + help = click.style( + "'fremor' is the main CLI for fremorizer. it houses the cmor subcommands.", + fg = 'cyan') +) +@click.option( '-v', '--verbose', default = 0, required = False, count = True, type = int, + help = "Increment logging verbosity from default (logging.WARNING) to logging.INFO. " + \ + "use -vv for logging.DEBUG. will be overridden by -q/--quiet" ) +@click.option( '-q', '--quiet', default = False, required = False, is_flag = True, type = bool, + help = "Set logging verbosity from default (logging.WARNING) to logging.ERROR, printing " + \ + "less output to screen. overrides -v[v]/--verbose" ) +@click.option( '-l', '--log_file', default = None, required = False, type = str, + help = 'Path to log file for all fremor calls, the output to screen will still print with the ' + \ + 'path specified. If the log file already exists, it is appended to.' ) +def fremor(verbose = 0, quiet = False, log_file = None): + ''' + entry point function to subgroup functions, setting global verbosity/logging formats that all + other routines will utilize + ''' + log_level = logging.WARNING # default + if verbose == 1: + log_level = logging.INFO # -v, more verbose than default + elif verbose == 2: + log_level = logging.DEBUG # -vv most verbose + + if quiet: + log_level = logging.ERROR # least verbose + + base_fre_logger=fre_logger.parent + base_fre_logger.setLevel(level = log_level) + fre_logger.debug('root fre_logger level set') + + # check if log_file arg was used + if log_file is not None: + fre_logger.debug('creating fre_file_handler for fre_logger') + fre_file_handler=logging.FileHandler(log_file, + mode='a',encoding='utf-8', + delay=False) + + fre_logger.debug('setting fre_file_handler logging format:') + fre_log_file_formatter=logging.Formatter(fmt=FORMAT) + fre_file_handler.setFormatter(fre_log_file_formatter) + + base_fre_logger.addHandler(fre_file_handler) + # first message that will appear in the log file if used + fre_logger.info('fre_file_handler added to base_fre_logger') + + fre_logger.debug('click entry-point function call done.') + + +@fremor.command() +@click.option("-y", "--yamlfile", type = str, + help = 'YAML file to be used for parsing', + required = True ) +@click.option("-e", "--experiment", type = str, + help = "Experiment name", + required = True ) +@click.option("-p", "--platform", type = str, + help = "Platform name", + required = True ) +@click.option("-t", "--target", type = str, + help = "Target name", + required = True ) +@click.option("-o", "--output", type = str, default = None, + help = "Output file if desired", required = False) +@click.option('--run_one', is_flag = True, default = False, + help=RUN_ONE_HELP, + required = False) +@click.option('--dry_run', is_flag = True, default = False, + help=DRY_RUN_HELP, + required = False) +@click.option('--start', type=str, default=None, + help = START_YEAR_HELP, + required = False) +@click.option('--stop', type=str, default=None, + help = STOP_YEAR_HELP, + required = False) +@click.option('--print_cli_call/--no-print_cli_call', default=True, + help = 'In dry-run mode, print the equivalent CLI invocation (default) ' + 'or the Python cmor_run_subtool() call.', + required = False) +def yaml(yamlfile, experiment, target, platform, output, run_one, dry_run, start, stop, print_cli_call): + """ + Processes a CMOR (Climate Model Output Rewriter) YAML configuration file. This function takes a YAML file + and various parameters related to a climate model experiment, and processes the YAML file using the CMOR + YAML subtool. + """ + cmor_yaml_subtool( + yamlfile = yamlfile, + exp_name = experiment, + target = target, + platform = platform, + output = output, + run_one_mode = run_one, + dry_run_mode = dry_run, + start = start, + stop = stop, + print_cli_call = print_cli_call + ) + + +@fremor.command() +@click.option("-l", "--varlist", type = str, + help=VARLIST_HELP, + required=False) +@click.option("-r", "--table_config_dir", type = str, + help="directory holding MIP tables to search for variables in var list", + required=True) +@click.option('-v', "--opt_var_name", type = str, + help=OPT_VAR_NAME_HELP, + required=False) +def find(varlist, table_config_dir, opt_var_name): #uncovered + ''' + loop over json table files in config_dir and show which tables contain variables in var list/ + the tool will also print what that table entry is expecting of that variable as well. if given + an opt_var_name in addition to varlist, only that variable name will be printed out. + accepts 3 arguments, two of the three required. + ''' + cmor_find_subtool( + json_var_list = varlist, + json_table_config_dir = table_config_dir, + opt_var_name = opt_var_name + ) + + +@fremor.command() +@click.option("-d", "--indir", type = str, + help="directory containing netCDF files. keys specified in json_var_list are local " + \ + "variable names used for targeting specific files in this directory", + required=True) +@click.option("-l", "--varlist", type = str, + help=VARLIST_HELP, + required=True) +@click.option("-r", "--table_config", type = str, + help="json file containing CMIP-compliant per-variable/metadata for specific " + \ + "MIP table. The MIP table can generally be identified by the specific " + \ + "filename (e.g. \'Omon\')", + required=True) +@click.option("-p", "--exp_config", type = str, + help="json file containing metadata dictionary for CMORization. this metadata is " + \ + "effectively appended to the final output file's header", + required=True) +@click.option("-o", "--outdir", type = str, + help="directory root that will contain the full output and output directory " + \ + "structure generated by the cmor module upon request.", + required=True) +@click.option('--run_one', is_flag = True, default = False, + help=RUN_ONE_HELP, + required = False) +@click.option('-v', "--opt_var_name", type = str, default = None, + help=OPT_VAR_NAME_HELP, + required=False) +@click.option('-g', '--grid_label', type = str, default = None, + help = 'label representing grid type of input data, e.g. "gn" for native or "gr" for regridded, ' + \ + 'replaces the "grid_label" field in the CMOR experiment configuration file. The label must ' + \ + 'be one of the entries in the MIP controlled-vocab file.', + required = False) +@click.option('--grid_desc', type = str, default = None, + help = 'description of grid indicated by grid label, replaces the "grid" field in the CMOR ' + \ + 'experiment configuration file.', + required = False) +@click.option('--nom_res', type = str, default = None, + help = 'nominal resolution indicated by grid and/or grid label, replaces the "nominal_resolution", ' + \ + 'replaces the "grid" field in the CMOR experiment configuration file. The entered string ' + \ + 'must be one of the entries in the MIP controlled-vocab file.', + required = False) +@click.option('--start', type=str, default=None, + help = START_YEAR_HELP, + required = False) +@click.option('--stop', type=str, default=None, + help = STOP_YEAR_HELP, + required = False) +@click.option('--calendar', type=str, default=None, + help = 'calendar type, e.g. 360_day, noleap, gregorian... etc', + required = False) +def run(indir, varlist, table_config, exp_config, outdir, run_one, opt_var_name, + grid_label, grid_desc, nom_res, start, stop, calendar): + # pylint: disable=unused-argument + """ + Rewrite climate model output files with CMIP-compliant metadata for down-stream publishing + """ + cmor_run_subtool( + indir = indir, + json_var_list = varlist, + json_table_config = table_config, + json_exp_config = exp_config, + outdir = outdir, + run_one_mode = run_one, + opt_var_name = opt_var_name, + grid = grid_desc, + grid_label = grid_label, + nom_res = nom_res, + start = start, + stop = stop, + calendar_type = calendar + ) + + +@fremor.command() +@click.option("-d", "--dir_targ", type=str, required=True, help="Target directory") +@click.option("-o", "--output_variable_list", type=str, required=True, help="Output variable list file") +@click.option("-t", "--mip_table", type=str, required=False, default=None, + help="Target MIP table for making variable list") +def varlist(dir_targ, output_variable_list, mip_table): + """ + Create a simple variable list from netCDF files in the target directory. + """ + make_simple_varlist(dir_targ = dir_targ, + output_variable_list = output_variable_list, + json_mip_table = mip_table) + + +@fremor.command() +@click.option("-p", "--pp_dir", type=str, required=True, + help="Root post-processing directory containing per-component subdirectories.") +@click.option("-t", "--mip_tables_dir", type=str, required=True, + help="Directory containing MIP table JSON files.") +@click.option("-m", "--mip_era", type=str, required=True, + help="MIP era identifier, e.g. 'cmip6' or 'cmip7'.") +@click.option("-e", "--exp_config", type=str, required=True, + help="Path to JSON experiment/input configuration file expected by CMOR.") +@click.option("-o", "--output_yaml", type=str, required=True, + help="Path for the output CMOR YAML configuration file.") +@click.option("-d", "--output_dir", type=str, required=True, + help="Root output directory for CMORized data.") +@click.option("-l", "--varlist_dir", type=str, required=True, + help="Directory in which per-component variable list JSON files are written.") +@click.option("--freq", type=str, default="monthly", + help="Temporal frequency string, e.g. 'monthly', 'daily'. Default 'monthly'.") +@click.option("--chunk", type=str, default="5yr", + help="Time chunk string, e.g. '5yr', '10yr'. Default '5yr'.") +@click.option("--grid", type=str, default="g99", + help="Grid label anchor name, e.g. 'g99', 'gn'. Default 'g99'.") +@click.option("--overwrite", is_flag=True, default=False, + help="Overwrite existing variable list files.") +@click.option("--calendar", type=str, default="noleap", + help="Calendar type, e.g. 'noleap', '360_day'. Default 'noleap'.") +def config(pp_dir, mip_tables_dir, mip_era, exp_config, output_yaml, + output_dir, varlist_dir, freq, chunk, grid, overwrite, calendar): + """ + Generate a CMOR YAML configuration file from a post-processing directory tree. + Scans pp_dir for components and time-series data, cross-references against MIP tables, + and writes a YAML configuration that 'fremor yaml' can consume. + """ + cmor_config_subtool( + pp_dir=pp_dir, + mip_tables_dir=mip_tables_dir, + mip_era=mip_era, + exp_config=exp_config, + output_yaml=output_yaml, + output_dir=output_dir, + varlist_dir=varlist_dir, + freq=freq, + chunk=chunk, + grid=grid, + overwrite=overwrite, + calendar_type=calendar + ) diff --git a/fremorizer/tests/__init__.py b/fremorizer/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/fremorizer/tests/test_cmor_config_subtool.py b/fremorizer/tests/test_cmor_config_subtool.py new file mode 100644 index 0000000..e33245c --- /dev/null +++ b/fremorizer/tests/test_cmor_config_subtool.py @@ -0,0 +1,110 @@ +''' +largely tests for fremorizer.cmor_config.cmor_config_subtool error conditions / messages +''' + +import tempfile +from pathlib import Path + +import pytest + +from fremorizer.cmor_config import cmor_config_subtool + +@pytest.fixture +def temp_dir(): + ''' fixture yielding a temporary directory ''' + with tempfile.TemporaryDirectory() as tmpdir: + yield tmpdir + + +def test_cmor_config_subtool_noppdir_err(temp_dir): # pylint: disable=redefined-outer-name + ''' pp_dir arg does not exist ''' + pp_dir_targ= Path(temp_dir) / 'foobar' + mip_tables_targ='' + mip_era_targ='' + exp_config_targ='' + with pytest.raises(FileNotFoundError, + match=f'pp_dir does not exist: {pp_dir_targ}'): + cmor_config_subtool(pp_dir=pp_dir_targ, + mip_tables_dir=mip_tables_targ, + mip_era=mip_era_targ, + exp_config=exp_config_targ, + output_yaml='', + output_dir='', + varlist_dir='', + ) + + +def test_cmor_config_subtool_notabledir_err(temp_dir): # pylint: disable=redefined-outer-name + ''' mip_tables_dir arg does not exist ''' + pp_dir_targ=Path(temp_dir) / 'foobar' + mip_tables_targ='fremorizer/tests/test_files/cmip7-cmor-tables/tablesDNE' + mip_era_targ='' + exp_config_targ='' + pp_dir_targ.mkdir(exist_ok=True,parents=True) + with pytest.raises(FileNotFoundError, + match=f'mip_tables_dir does not exist: {mip_tables_targ}'): + cmor_config_subtool(pp_dir=pp_dir_targ, + mip_tables_dir=mip_tables_targ, + mip_era=mip_era_targ, + exp_config=exp_config_targ, + output_yaml='', + output_dir='', + varlist_dir='', + ) + + +def test_cmor_config_subtool_noexpcfg_err(temp_dir): # pylint: disable=redefined-outer-name + ''' exp_config arg does not exist ''' + pp_dir_targ=Path(temp_dir) / 'foobar' + mip_tables_targ='fremorizer/tests/test_files/cmip7-cmor-tables/tables' + mip_era_targ='' + exp_config_targ='fremorizer/tests/test_files/DNE_CMOR_CMIP7_input_example.json' + pp_dir_targ.mkdir(exist_ok=True,parents=True) + with pytest.raises(FileNotFoundError, + match=f'exp_config does not exist: {exp_config_targ}'): + cmor_config_subtool(pp_dir=pp_dir_targ, + mip_tables_dir=mip_tables_targ, + mip_era=mip_era_targ, + exp_config=exp_config_targ, + output_yaml='', + output_dir='', + varlist_dir='', + ) + + +def test_cmor_config_subtool_nomip6_tables_in_mip7_tables_err(temp_dir): # pylint: disable=redefined-outer-name + ''' trying to target mip7 tables for mip6 ''' + pp_dir_targ= Path(temp_dir) / 'foobar' + mip_tables_targ='fremorizer/tests/test_files/cmip7-cmor-tables/tables' + mip_era_targ='cmip6' + exp_config_targ='fremorizer/tests/test_files/CMOR_CMIP7_input_example.json' + pp_dir_targ.mkdir(exist_ok=True,parents=True) + with pytest.raises(ValueError, + match=f'no MIP tables found in {mip_tables_targ} for era {mip_era_targ} after filtering'): + cmor_config_subtool(pp_dir=pp_dir_targ, + mip_tables_dir=mip_tables_targ, + mip_era=mip_era_targ, + exp_config=exp_config_targ, + output_yaml='', + output_dir='', + varlist_dir='', + ) + + +def test_cmor_config_subtool_nomip7_tables_in_mip6_tables_err(temp_dir): # pylint: disable=redefined-outer-name + ''' trying to target mip6 tables for mip7 ''' + pp_dir_targ= Path(temp_dir) / 'foobar' + mip_tables_targ='fremorizer/tests/test_files/cmip6-cmor-tables/Tables' + mip_era_targ='cmip7' + exp_config_targ='fremorizer/tests/test_files/CMOR_input_example.json' + pp_dir_targ.mkdir(exist_ok=True,parents=True) + with pytest.raises(ValueError, + match=f'no MIP tables found in {mip_tables_targ} for era {mip_era_targ} after filtering'): + cmor_config_subtool(pp_dir=pp_dir_targ, + mip_tables_dir=mip_tables_targ, + mip_era=mip_era_targ, + exp_config=exp_config_targ, + output_yaml='', + output_dir='', + varlist_dir='', + ) diff --git a/fremorizer/tests/test_cmor_find_subtool.py b/fremorizer/tests/test_cmor_find_subtool.py new file mode 100644 index 0000000..1156501 --- /dev/null +++ b/fremorizer/tests/test_cmor_find_subtool.py @@ -0,0 +1,75 @@ +''' +tests for fremorizer.cmor_finder.cmor_find_subtool, +mostly +''' + +import json +import tempfile +from pathlib import Path + +import pytest + +from fremorizer.cmor_finder import make_simple_varlist, cmor_find_subtool + +@pytest.fixture +def temp_dir(): + ''' simple pytest fixture for providing temp directory ''' + with tempfile.TemporaryDirectory() as tmpdir: + yield tmpdir + + +def test_make_simple_varlist(temp_dir): + ''' + quick tests of make_simple_varlist + ''' + # Create some dummy netCDF files + nc_files = ["test.20230101.var1.nc", "test.20230101.var2.nc", "test.20230101.var3.nc"] + assert Path(temp_dir).exists() + for nc_file in nc_files: + Path(temp_dir, nc_file).touch() + + output_file = Path(temp_dir, "varlist.json") + make_simple_varlist(temp_dir, output_file) + + # Check if the output file is created + assert output_file.exists() + + # Check the contents of the output file + with open(output_file, 'r') as f: + var_list = json.load(f) + + expected_var_list = { + "var1": "var1", + "var2": "var2", + "var3": "var3" + } + assert var_list == expected_var_list + + +def test_find_subtool_no_json_dir_err(temp_dir): + ''' test json_table_config_dir does not exist error ''' + target_dir_DNE = Path(temp_dir) / 'foo' + assert not target_dir_DNE.exists(), 'target dir should not exist for this test' + with pytest.raises(OSError, match=f'ERROR directory {target_dir_DNE} does not exist! exit.'): + cmor_find_subtool(json_var_list=None, + json_table_config_dir=str(target_dir_DNE), + opt_var_name=None) + + +def test_find_subtool_no_json_files_in_dir_err(temp_dir): + ''' test json_table_config_dir has no files in it error ''' + target_dir = Path(temp_dir) / 'foo' + target_dir.mkdir(exist_ok=False) + assert target_dir.is_dir() and target_dir.exists(), "temp dir directory creation failed, inspect code" + with pytest.raises(OSError, match=f'ERROR directory {target_dir} contains no JSON files, exit.'): + cmor_find_subtool(json_var_list=None, + json_table_config_dir=str(target_dir), + opt_var_name=None) + + +def test_find_subtool_no_varlist_no_optvarname_err(temp_dir): + ''' test no opt_var_name AND no varlist error ''' + with pytest.raises(ValueError, match='ERROR: no opt_var_name given but also no content in variable list!!! exit!'): + cmor_find_subtool(json_var_list=None, + json_table_config_dir='fremorizer/tests/test_files/cmip6-cmor-tables/Tables', + opt_var_name=None) diff --git a/fremorizer/tests/test_cmor_finder_make_simple_varlist.py b/fremorizer/tests/test_cmor_finder_make_simple_varlist.py new file mode 100644 index 0000000..8cd9bb7 --- /dev/null +++ b/fremorizer/tests/test_cmor_finder_make_simple_varlist.py @@ -0,0 +1,282 @@ +''' +tests for fremorizer.cmor_finder.make_simple_varlist +''' + +import json +from unittest.mock import patch + +import pytest + +from fremorizer.cmor_finder import make_simple_varlist + + +@pytest.fixture +def temp_netcdf_dir(tmp_path): + """ + Fixture to create a temporary directory with sample NetCDF files. + """ + # Create sample NetCDF filenames following the expected pattern + netcdf_files = [ + "model.19900101.temp.nc", + "model.19900101.salt.nc", + "model.19900101.velocity.nc" + ] + + for filename in netcdf_files: + file_path = tmp_path / filename + file_path.touch() # Create empty files for testing + + return tmp_path + + +@pytest.fixture +def temp_netcdf_dir_single_file(tmp_path): + """ + Fixture to create a temporary directory with a single NetCDF file. + """ + file_path = tmp_path / "model.19900101.temp.nc" + file_path.touch() + return tmp_path + + +@pytest.fixture +def empty_dir(tmp_path): + """ + Fixture to create an empty temporary directory. + """ + return tmp_path + + +def test_make_simple_varlist_success(temp_netcdf_dir, tmp_path): + """ + Test successful creation of variable list from NetCDF files. + """ + # Arrange + output_file = tmp_path / "varlist.json" + + # Act + result = make_simple_varlist(str(temp_netcdf_dir), str(output_file)) + + # Assert + assert result is not None + assert isinstance(result, dict) + assert "temp" in result + assert "salt" in result + assert "velocity" in result + assert result["temp"] == "temp" + assert result["salt"] == "salt" + assert result["velocity"] == "velocity" + + # Verify output file was created + assert output_file.exists() + with open(output_file, "r") as f: + saved_data = json.load(f) + assert saved_data == result + + +def test_make_simple_varlist_return_value_only(temp_netcdf_dir): + """ + Test make_simple_varlist with output_variable_list=None returns var_list. + """ + # Act + result = make_simple_varlist(str(temp_netcdf_dir), None) + + # Assert + assert result is not None + assert isinstance(result, dict) + assert "temp" in result + assert "salt" in result + assert "velocity" in result + + +def test_make_simple_varlist_single_file_warning(temp_netcdf_dir_single_file, tmp_path): + """ + Test warning when only one file is found. + """ + # Arrange + output_file = tmp_path / "varlist.json" + + # Act + result = make_simple_varlist(str(temp_netcdf_dir_single_file), str(output_file)) + + # Assert + assert result is not None + assert isinstance(result, dict) + assert "temp" in result + assert result["temp"] == "temp" + + +def test_make_simple_varlist_no_files(empty_dir): + """ + Test behavior when no NetCDF files are found in directory. + """ + # Act + result = make_simple_varlist(str(empty_dir), None) + + # Assert - function should return None when no files found + assert result is None + + +def test_make_simple_varlist_invalid_output_path(temp_netcdf_dir): + """ + Test OSError when output file cannot be written. + """ + # Arrange - try to write to a directory that doesn't exist + invalid_output_path = "/nonexistent_directory/varlist.json" + + # Act & Assert + with pytest.raises(OSError, match="output variable list created but cannot be written"): + make_simple_varlist(str(temp_netcdf_dir), invalid_output_path) + + +def test_make_simple_varlist_no_matching_pattern(tmp_path): + """ + Test behavior when files exist but don't match the expected pattern. + """ + # Arrange - create files that don't follow the expected pattern + (tmp_path / "random_file.txt").touch() + (tmp_path / "another_file.nc").touch() # Missing datetime pattern + + # Act + result = make_simple_varlist(str(tmp_path), None) + + # Assert - should return None when no matching files found + assert result is None + + +# ---- duplicate var_name skip coverage ---- +def test_make_simple_varlist_deduplicates(tmp_path): + """ + When multiple files share the same var_name across different datetimes, + the result should contain the variable only once. Variables that only + appear at a single datetime are still included. + """ + # Two files with var_name "temp" and one with "salt" + (tmp_path / "model.19900101.temp.nc").touch() + (tmp_path / "model.19900201.temp.nc").touch() # duplicate var_name + (tmp_path / "model.19900101.salt.nc").touch() + + result = make_simple_varlist(str(tmp_path), None) + + assert result is not None + assert result == {"temp": "temp", "salt": "salt"} + + +# ---- mip-table filtering coverage ---- +def test_make_simple_varlist_mip_table_filter(tmp_path): + """ + When a json_mip_table is provided, only variables present in the MIP table + should appear in the result. + """ + # create data files + (tmp_path / "model.19900101.sos.nc").touch() + (tmp_path / "model.19900101.notinmip.nc").touch() + + # create a minimal MIP table with only "sos" + mip_table = tmp_path / "Omon.json" + mip_table.write_text(json.dumps({ + "variable_entry": { + "sos": {"frequency": "mon"} + } + })) + + result = make_simple_varlist(str(tmp_path), None, json_mip_table=str(mip_table)) + + assert result is not None + assert "sos" in result + assert "notinmip" not in result + + +# ---- no files matching search pattern ---- +def test_make_simple_varlist_no_files_matching_pattern(tmp_path): + """ + When glob finds no *.*.nc files in the directory the function should + return None. Covers the 'if not all_nc_files' early return by patching + glob.glob to return an empty list. + """ + # Create a file for the probe to land on (but the patch overrides it) + probe_file = tmp_path / "model.19900101.temp.nc" + probe_file.touch() + + # Patch glob.glob to return empty list + with patch('fremorizer.cmor_finder.glob.glob', return_value=[]): + result = make_simple_varlist(str(tmp_path), None) + + assert result is None + + +# ---- single file warning path ---- +def test_make_simple_varlist_single_file_hits_warning(tmp_path): + """ + When exactly one file matches the search pattern, the function should + log a warning and still return the variable. + Covers the 'elif len(files) == 1' branch. + """ + (tmp_path / "model.19900101.salinity.nc").touch() + + result = make_simple_varlist(str(tmp_path), None) + + assert result is not None + assert result == {"salinity": "salinity"} + + +# ---- duplicate var_name skip with datetime grouping ---- +def test_make_simple_varlist_dedup_across_datetimes(tmp_path): + """ + Files with different datetime stamps but the same variable name + should be de-duplicated so the variable appears only once. + All files across all datetimes are scanned; duplicate var names + are collapsed by dict assignment. + """ + (tmp_path / "ocean.19900101.tos.nc").touch() + (tmp_path / "ocean.19900201.tos.nc").touch() + (tmp_path / "ocean.19900101.sos.nc").touch() + (tmp_path / "ocean.19900201.sos.nc").touch() + + # All four files are scanned; tos and sos each appear twice but + # dict assignment deduplicates them. + result = make_simple_varlist(str(tmp_path), None) + + assert result is not None + assert result == {"tos": "tos", "sos": "sos"} + + +# ---- mip table filtering: no variables match ---- +def test_make_simple_varlist_mip_table_no_match(tmp_path): + """ + When a MIP table is provided but none of the file variables are in it, + the result should be an empty dict (quick_vlist stays empty → 'no + variables in target mip table found' warning, var_list stays {}). + """ + (tmp_path / "model.19900101.fake_var.nc").touch() + + mip_table = tmp_path / "table.json" + mip_table.write_text(json.dumps({ + "variable_entry": { + "sos": {"frequency": "mon"} + } + })) + + result = make_simple_varlist(str(tmp_path), None, json_mip_table=str(mip_table)) + + # No variables matched + assert result is None + + +# ---- variable only present at a minority datetime is still returned ---- +def test_make_simple_varlist_minority_datetime_var_included(tmp_path): + """ + A variable that only appears at one datetime should still be returned + even when most files belong to a different datetime. Scanning all files + (not just those at the most-common datetime) guarantees this. + """ + # "temp" appears four times at 19900101; "salt" appears only once at 19900201. + for i in range(1, 5): + (tmp_path / f"model.1990010{i}.temp.nc").touch() + (tmp_path / "model.19900201.salt.nc").touch() + + result = make_simple_varlist(str(tmp_path), None) + + assert result is not None + assert "temp" in result + assert "salt" in result # must not be silently dropped diff --git a/fremorizer/tests/test_cmor_helpers.py b/fremorizer/tests/test_cmor_helpers.py new file mode 100644 index 0000000..a835f86 --- /dev/null +++ b/fremorizer/tests/test_cmor_helpers.py @@ -0,0 +1,438 @@ +''' +tests for fremorizer helper functions in cmor_helpers +''' + +import json +from pathlib import Path + +import numpy as np +import pytest + +from fremorizer.cmor_helpers import ( find_statics_file, print_data_minmax, + find_gold_ocean_statics_file, + create_lev_bnds, get_iso_datetime_ranges, iso_to_bronx_chunk, + create_tmp_dir, get_json_file_data, + update_grid_and_label, get_bronx_freq_from_mip_table, #update_outpath, + filter_brands ) + +def test_iso_to_bronx_chunk(): + ''' tests value error raising by iso_to_bronx_chunk ''' + with pytest.raises(ValueError, + match='problem with converting to bronx chunk from the cmor chunk. check cmor_yamler.py'): + iso_to_bronx_chunk('foo') + +def test_find_statics_file_success(): + ''' what happens when no statics file is found given a bronx directory structure ''' + target_file = 'fremorizer/tests/test_files/ascii_files/mock_archive/' + \ + 'USER/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/' + \ + 'gfdl.ncrc5-intel23-prod-openmp/pp/ocean_monthly/ts/monthly/5yr/ocean_monthly.000101-000102.sos.nc' + if not Path(target_file).exists(): + Path(target_file).touch() + assert Path(target_file).exists() + + expected_answer_statics_file = 'fremorizer/tests/test_files/ascii_files/mock_archive/' + \ + 'USER/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/' + \ + 'gfdl.ncrc5-intel23-prod-openmp/pp/ocean_monthly/ocean_monthly.static.nc' + if not Path(expected_answer_statics_file).exists(): + Path(expected_answer_statics_file).touch() + assert Path(expected_answer_statics_file).exists + + statics_file = find_statics_file( bronx_file_path = target_file + ) + assert Path(statics_file).exists() + assert statics_file == expected_answer_statics_file + + +def test_find_statics_file_nothing_found(): + ''' what happens when a statics file is found given a bronx directory structure ''' + statics_file = find_statics_file( + bronx_file_path = 'fremorizer/tests/test_files/ascii_files/' + \ + 'mock_archive/USER/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/' + \ + 'gfdl.ncrc5-intel23-prod-openmp/pp/land/ts/monthly/5yr/land.000101-000512.lai.nc' ) + assert statics_file is None + + +def test_print_data_minmax_no_exception_case1(): + ''' checks to make sure this doesn't raise an exception ''' + print_data_minmax(None, None) + +def test_print_data_minmax_no_exception_case2(): + ''' checks to make sure this doesn't raise an exception ''' + print_data_minmax(np.ma.core.MaskedArray( data=(0, 10, 20, 30) ), None) + +def test_print_data_minmax_no_exception_case3(): + ''' checks to make sure this doesn't raise an exception ''' + print_data_minmax(np.ma.core.MaskedArray( data=(0, 10, 20, 30) ), 'foo') + + +# ---- find_gold_ocean_statics_file tests ---- + +def test_find_gold_ocean_statics_file_none_arg(): + ''' put_copy_here=None should return None immediately ''' + result = find_gold_ocean_statics_file(put_copy_here=None) + assert result is None + + +def test_find_gold_ocean_statics_file_archive_missing(tmp_path): + ''' + when the archive gold file does not exist on disk (i.e. not at PPAN), + the function should create the local directory tree but return None + because there's nothing to copy. + ''' + from fremorizer.cmor_constants import ARCHIVE_GOLD_DATA_DIR, CMIP7_GOLD_OCEAN_FILE_STUB # pylint: disable=import-outside-toplevel + gold_file = f'{ARCHIVE_GOLD_DATA_DIR}/{CMIP7_GOLD_OCEAN_FILE_STUB}' + + result = find_gold_ocean_statics_file(put_copy_here=str(tmp_path)) + # on dev boxes the archive path won't exist, so we get None + if result is None: + if Path(gold_file).is_file(): + assert False, ( + f'gold file exists at {gold_file} but function returned None. error!' + ) + else: + # if we happen to be at PPAN and it succeeded, just check it's a real file + assert Path(result).is_file() + assert True + + +def test_find_gold_ocean_statics_file_mock_copy(tmp_path): + ''' + exercise the full copy path by creating a fake archive gold file in tmp_path + and monkeypatching ARCHIVE_GOLD_DATA_DIR so the function finds it. + ''' + import fremorizer.cmor_helpers as _helpers_mod # pylint: disable=import-outside-toplevel + import fremorizer.cmor_constants as _const_mod # pylint: disable=import-outside-toplevel + + # build a fake archive layout: /gold/datasets/OM5_025/.../ocean_static.nc + fake_archive_root = tmp_path / 'fake_archive' / 'gold' / 'datasets' + fake_gold_file = fake_archive_root / 'OM5_025' / 'ocean_mosaic_v20250916_unpacked' / 'ocean_static.nc' + fake_gold_file.parent.mkdir(parents=True, exist_ok=True) + fake_gold_file.write_text('placeholder') + + staging_dir = tmp_path / 'staging' + + # monkeypatch the constant in both cmor_constants and cmor_helpers (where it's already imported) + original_val = _const_mod.ARCHIVE_GOLD_DATA_DIR + try: + _const_mod.ARCHIVE_GOLD_DATA_DIR = str(fake_archive_root) + _helpers_mod.ARCHIVE_GOLD_DATA_DIR = str(fake_archive_root) + result = find_gold_ocean_statics_file(put_copy_here=str(staging_dir)) + finally: + _const_mod.ARCHIVE_GOLD_DATA_DIR = original_val + _helpers_mod.ARCHIVE_GOLD_DATA_DIR = original_val + + assert result is not None + assert Path(result).is_file() + assert 'ocean_static.nc' in result + + +# ---- create_lev_bnds failure case ---- + +def test_create_lev_bnds_length_mismatch(): + ''' create_lev_bnds should raise ValueError when len(with_these) != len(bound_these)+1 ''' + bound_these = np.array([10.0, 20.0, 30.0]) + with_these = np.array([5.0, 15.0]) # wrong: should be len 4 (=3+1) + with pytest.raises(ValueError, match='failed creating bnds'): + create_lev_bnds(bound_these=bound_these, with_these=with_these) + + +def test_create_lev_bnds_length_mismatch_too_long(): + ''' same check, but with_these is too long instead of too short ''' + bound_these = np.array([10.0, 20.0]) + with_these = np.array([5.0, 15.0, 25.0, 35.0]) # wrong: should be len 3 (=2+1) + with pytest.raises(ValueError, match='failed creating bnds'): + create_lev_bnds(bound_these=bound_these, with_these=with_these) + + +# ---- get_iso_datetime_ranges with stop_yr ---- + +# helper filenames that look like FRE-bronx time-series files +_SAMPLE_FILENAMES = [ + 'ocean_monthly.19900101-19941231.sos.nc', + 'ocean_monthly.19950101-19991231.sos.nc', + 'ocean_monthly.20000101-20041231.sos.nc', + 'ocean_monthly.20050101-20091231.sos.nc', + 'ocean_monthly.20100101-20141231.sos.nc', +] + + +def test_get_iso_datetime_ranges_no_filter(): + ''' all 5 date ranges should appear when start/stop are None ''' + result = [] + get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, + iso_daterange_arr=result) + assert len(result) == 5 + assert '19900101-19941231' in result + assert '20100101-20141231' in result + + +def test_get_iso_datetime_ranges_with_stop(): + ''' only date ranges whose end-year <= 2004 should survive ''' + result = [] + get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, + iso_daterange_arr=result, + stop='2004') + # ranges ending in 1994, 1999, 2004 qualify; 2009, 2014 do not + assert '19900101-19941231' in result + assert '19950101-19991231' in result + assert '20000101-20041231' in result + assert '20050101-20091231' not in result + assert '20100101-20141231' not in result + assert len(result) == 3 + + +def test_get_iso_datetime_ranges_with_start(): + ''' only date ranges whose start-year >= 2000 should survive ''' + result = [] + get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, + iso_daterange_arr=result, + start='2000') + assert '19900101-19941231' not in result + assert '19950101-19991231' not in result + assert '20000101-20041231' in result + assert '20050101-20091231' in result + assert '20100101-20141231' in result + assert len(result) == 3 + + +def test_get_iso_datetime_ranges_with_start_and_stop(): + ''' start=1995 stop=2004 should give exactly two ranges ''' + result = [] + get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, + iso_daterange_arr=result, + start='1995', + stop='2004') + assert result == ['19950101-19991231', '20000101-20041231'] + + +def test_get_iso_datetime_ranges_none_arr_raises(): + ''' passing iso_daterange_arr=None should raise ValueError ''' + with pytest.raises(ValueError, match='requires the list'): + get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, + iso_daterange_arr=None) + + +def test_get_iso_datetime_ranges_no_matches_raises(): + ''' if the filter excludes everything, ValueError should be raised ''' + result = [] + with pytest.raises(ValueError, match='length 0'): + get_iso_datetime_ranges(var_filenames=_SAMPLE_FILENAMES, + iso_daterange_arr=result, + start='2050', + stop='2060') + + +# ---- create_tmp_dir tests ---- + +def test_create_tmp_dir_success(tmp_path): + ''' create_tmp_dir should create a tmp/ subdirectory and return its path ''' + result = create_tmp_dir(outdir=str(tmp_path)) + assert Path(result).is_dir() + assert result.endswith('/CMOR_tmp') + + +def test_create_tmp_dir_with_exp_config(tmp_path): + ''' when json_exp_config has an outpath key, a subdirectory should also be created ''' + exp_config = tmp_path / 'exp_config.json' + exp_config.write_text(json.dumps({"outpath": "CMIP7/output"})) + result = create_tmp_dir(outdir=str(tmp_path), json_exp_config=str(exp_config)) + assert Path(result).is_dir() + assert Path(result, 'CMIP7/output').is_dir() + + +def test_create_tmp_dir_oserror(tmp_path): + ''' create_tmp_dir should raise OSError when directory creation fails ''' + # /dev/null is not a directory; we can't mkdir inside it + with pytest.raises(OSError, match='problem creating tmp output directory'): + create_tmp_dir(outdir='/dev/null/impossible_path') + + +# ---- get_json_file_data tests ---- + +def test_get_json_file_data_success(tmp_path): + ''' should load and return JSON content ''' + f = tmp_path / 'data.json' + payload = {"key": "value", "num": 42} + f.write_text(json.dumps(payload)) + assert get_json_file_data(str(f)) == payload + + +def test_get_json_file_data_nonexistent(): + ''' should raise FileNotFoundError for a missing file ''' + with pytest.raises(FileNotFoundError, match='cannot be opened'): + get_json_file_data('/nonexistent/path/file.json') + + +def test_get_json_file_data_invalid_json(tmp_path): + ''' should raise FileNotFoundError (wrapping JSONDecodeError) for invalid JSON ''' + f = tmp_path / 'bad.json' + f.write_text('NOT JSON {{{{') + with pytest.raises(FileNotFoundError, match='cannot be opened'): + get_json_file_data(str(f)) + + +# ---- update_grid_and_label None-args test ---- + +def test_update_grid_and_label_none_grid_label(tmp_path): + ''' should raise ValueError when new_grid_label is None ''' + f = tmp_path / 'exp.json' + f.write_text(json.dumps({"grid_label": "gr", "grid": "g", "nominal_resolution": "50 km"})) + with pytest.raises(ValueError): + update_grid_and_label(str(f), None, "new_grid", "100 km") + + +def test_update_grid_and_label_none_grid(tmp_path): + ''' should raise ValueError when new_grid is None ''' + f = tmp_path / 'exp.json' + f.write_text(json.dumps({"grid_label": "gr", "grid": "g", "nominal_resolution": "50 km"})) + with pytest.raises(ValueError): + update_grid_and_label(str(f), "gn", None, "100 km") + + +def test_update_grid_and_label_none_nom_res(tmp_path): + ''' should raise ValueError when new_nom_res is None ''' + f = tmp_path / 'exp.json' + f.write_text(json.dumps({"grid_label": "gr", "grid": "g", "nominal_resolution": "50 km"})) + with pytest.raises(ValueError): + update_grid_and_label(str(f), "gn", "new_grid", None) + + +# ---- get_bronx_freq_from_mip_table tests ---- + +def test_get_bronx_freq_from_mip_table_success(tmp_path): + ''' should return the bronx-equivalent frequency for a valid table ''' + table = { + "variable_entry": { + "sos": {"frequency": "mon", "other": "stuff"} + } + } + f = tmp_path / 'Omon.json' + f.write_text(json.dumps(table)) + assert get_bronx_freq_from_mip_table(str(f)) == "monthly" + +def test_get_bronx_freq_from_mip_table_no_freq(tmp_path): + ''' should raise bronx-equivalent frequency for a valid table ''' + table = { + "variable_entry": { + "sos": {"other": "stuff"} + } + } + f = tmp_path / 'Omon.json' + f.write_text(json.dumps(table)) + with pytest.raises(KeyError, + match='no frequency in table under variable_entry. this may be a CMIP7 table.'): + get_bronx_freq_from_mip_table(str(f)) + +def test_get_bronx_freq_from_mip_table_invalid_freq(tmp_path): + ''' should raise KeyError when the table frequency is not a valid MIP frequency ''' + table = { + "variable_entry": { + "sos": {"frequency": "bogus_freq"} + } + } + f = tmp_path / 'Obogus.json' + f.write_text(json.dumps(table)) + with pytest.raises(KeyError, match='not a valid MIP frequency'): + get_bronx_freq_from_mip_table(str(f)) + +## ---- update_outpath tests ---- +# +#def test_update_outpath_none_json_path(): +# ''' should raise ValueError when json_file_path is None ''' +# with pytest.raises(ValueError): +# update_outpath(None, '/some/output/path') +# +# +#def test_update_outpath_none_output_path(tmp_path): +# ''' should raise ValueError when output_file_path is None ''' +# f = tmp_path / 'exp.json' +# f.write_text(json.dumps({"outpath": "/old/path"})) +# with pytest.raises(ValueError): +# update_outpath(str(f), None) + + +# ---- filter_brands tests ---- + +def _make_mip_var_cfgs(var_brands_dims): + '''helper: build a minimal mip_var_cfgs dict from {mip_key: dims_string} pairs''' + return {"variable_entry": {k: {"dimensions": v} for k, v in var_brands_dims.items()}} + + +def test_filter_brands_time_filter_selects_mean(): + ''' brand with time (mean) should be selected when input has time_bnds ''' + mip = _make_mip_var_cfgs({ + "sos_mean-2d": "longitude latitude time", + "sos_inst-2d": "longitude latitude time1", + }) + result = filter_brands( + brands=["mean-2d", "inst-2d"], + target_var="sos", + mip_var_cfgs=mip, + has_time_bnds=True, + input_vert_dim=0, + ) + assert result == "mean-2d" + + +def test_filter_brands_time_filter_selects_inst(): + ''' brand with time1 (instantaneous) should be selected when input lacks time_bnds ''' + mip = _make_mip_var_cfgs({ + "sos_mean-2d": "longitude latitude time", + "sos_inst-2d": "longitude latitude time1", + }) + result = filter_brands( + brands=["mean-2d", "inst-2d"], + target_var="sos", + mip_var_cfgs=mip, + has_time_bnds=False, + input_vert_dim=0, + ) + assert result == "inst-2d" + + +def test_filter_brands_vertical_filter(): + ''' vertical filter should select the brand whose MIP dims contain the expected vert dim ''' + mip = _make_mip_var_cfgs({ + "temp_mean-3d-native-sea": "longitude latitude olevel time", + "temp_mean-2d": "longitude latitude time", + }) + result = filter_brands( + brands=["mean-3d-native-sea", "mean-2d"], + target_var="temp", + mip_var_cfgs=mip, + has_time_bnds=True, + input_vert_dim="z_l", + ) + assert result == "mean-3d-native-sea" + + +def test_filter_brands_all_eliminated(): + ''' should raise ValueError when all brands are filtered out ''' + mip = _make_mip_var_cfgs({ + "sos_a": "longitude latitude time1", + "sos_b": "longitude latitude time1", + }) + with pytest.raises(ValueError, match='none survived'): + filter_brands( + brands=["a", "b"], + target_var="sos", + mip_var_cfgs=mip, + has_time_bnds=True, + input_vert_dim=0, + ) + + +def test_filter_brands_multiple_remain(): + ''' should raise ValueError when multiple brands survive filtering ''' + mip = _make_mip_var_cfgs({ + "sos_a": "longitude latitude time", + "sos_b": "longitude latitude time", + }) + with pytest.raises(ValueError, match='remain for sos'): + filter_brands( + brands=["a", "b"], + target_var="sos", + mip_var_cfgs=mip, + has_time_bnds=True, + input_vert_dim=0, + ) diff --git a/fremorizer/tests/test_cmor_helpers_update_calendar.py b/fremorizer/tests/test_cmor_helpers_update_calendar.py new file mode 100644 index 0000000..76d9807 --- /dev/null +++ b/fremorizer/tests/test_cmor_helpers_update_calendar.py @@ -0,0 +1,113 @@ +''' +tests for fremorizer.cmor_helpers.update_calendar_type +''' +import json + +import pytest + +from fremorizer.cmor_helpers import update_calendar_type + +@pytest.fixture +def temp_json_file(tmp_path): + """ + Fixture to create a temporary JSON file for testing. + + Args: + tmp_path: pytest's fixture for temporary directories. + + Returns: + Path to the temporary JSON file. + """ + # Sample data for testing + test_json_content = { + "calendar": "original_calendar_type", + "other_field": "some_value" + } + + json_file = tmp_path / "test_file.json" + with open(json_file, "w", encoding="utf-8") as file: + json.dump(test_json_content, file, indent=4) + return json_file + +def test_update_calendar_type_success(temp_json_file): + """ + Test successful update of 'grid_label' and 'grid' fields. + """ + # Arrange + new_calendar_type = "365_day" + + # Act + update_calendar_type(temp_json_file, new_calendar_type) + + # Assert + with open(temp_json_file, "r", encoding="utf-8") as file: + data = json.load(file) + assert data["calendar"] == new_calendar_type + assert data["other_field"] == "some_value" + +def test_update_calendar_type_valerr_raise(temp_json_file): + """ + Test error raising when the input calendar is None + """ + with pytest.raises(ValueError): + update_calendar_type(temp_json_file, None) + +def test_update_calendar_type_unknown_err(): + """ + Test raising an exception not caught by the other ones + """ + bad_path = 12345 + with pytest.raises(Exception): + update_calendar_type( bad_path, '365_day') + +@pytest.fixture +def temp_keyerr_json_file(tmp_path): + """ + Fixture to create a temporary JSON file for testing. + + Args: + tmp_path: pytest's fixture for temporary directories. + + Returns: + Path to the temporary JSON file. + """ + # Sample data for testing + test_json_content = { + "clendar": "original_calendar_type", #oops spelling error # cspell:disable-line + "other_field": "some_value" + } + + json_file = tmp_path / "test_file.json" + with open(json_file, "w", encoding="utf-8") as file: + json.dump(test_json_content, file, indent=4) + return json_file + +def test_update_calendar_type_keyerror_raise(temp_keyerr_json_file): + """ + Test error raising when the calendar key doesn't exist + """ + with pytest.raises(KeyError): + update_calendar_type(temp_keyerr_json_file,'365_day') + +@pytest.fixture +def temp_jsondecodeerr_json_file(tmp_path): + # Create a file with invalid JSON content + invalid_json_file = tmp_path / "invalid.json" + invalid_content = '{ "calendar": "original_calendar_type", "other_field": "some_value" ' # missing closing } + with open(invalid_json_file, "w", encoding="utf-8") as f: + f.write(invalid_content) + return invalid_json_file + +def test_update_calendar_type_jsondecode_raise(temp_jsondecodeerr_json_file): + """ + Test raising a JSONDecodeError + """ + with pytest.raises(json.JSONDecodeError): + update_calendar_type(temp_jsondecodeerr_json_file, '365_day') + +def test_update_calendar_type_jsonDNE_raise(): + """ + Test error raising when the input experiment json doesn't exist + """ + with pytest.raises(FileNotFoundError): + update_calendar_type('DOES_NOT_EXIST.json','365_day') diff --git a/fremorizer/tests/test_cmor_helpers_update_grid_label.py b/fremorizer/tests/test_cmor_helpers_update_grid_label.py new file mode 100644 index 0000000..ae144c2 --- /dev/null +++ b/fremorizer/tests/test_cmor_helpers_update_grid_label.py @@ -0,0 +1,148 @@ +''' +unit tests for cmor_helpers.update_grid_and_label +''' + +import json + +from pathlib import Path +import pytest + +from fremorizer.cmor_helpers import update_grid_and_label + +# Sample data for testing +TEST_JSON_CONTENT = { + "grid_label": "original_label", + "grid": "original_grid", + "nominal_resolution": "original_nom_res", + "other_field": "some_value" +} + +@pytest.fixture +def temp_json_file(tmp_path): + """ + Fixture to create a temporary JSON file for testing. + + Args: + tmp_path: pytest's fixture for temporary directories. + + Returns: + Path to the temporary JSON file. + """ + json_file = tmp_path / "test_file.json" + with open(json_file, "w", encoding="utf-8") as file: + json.dump(TEST_JSON_CONTENT, file, indent=4) + return json_file + +def test_update_grid_label_and_grid_success(temp_json_file): + """ + Test successful update of 'grid_label' and 'grid' fields. + """ + # Arrange + new_grid = "updated_grid" + new_grid_label = "updated_label" + new_nom_res = "updated_nom_res" + + # Act + update_grid_and_label(temp_json_file, new_grid_label, new_grid, new_nom_res) + + # Assert + with open(temp_json_file, "r", encoding="utf-8") as file: + data = json.load(file) + assert data["grid"] == new_grid + assert data["grid_label"] == new_grid_label + assert data["nominal_resolution"] == new_nom_res + assert data["other_field"] == "some_value" # Ensure other fields are untouched + +def test_missing_nom_res_field(temp_json_file): + """ + Test behavior when the 'nominal_resolution' field is missing in the JSON file. + """ + # Arrange + with open(temp_json_file, "r+", encoding="utf-8") as file: + data = json.load(file) + del data["nominal_resolution"] # Remove the 'nominal_resolution' field + file.seek(0) + json.dump(data, file, indent=4) + file.truncate() + + new_grid_label = "updated_label" + new_grid = "updated_grid" + new_nom_res = "updated_nom_res" + + # Act & Assert + with pytest.raises( + KeyError, + match='"Error updating \'nominal_resolution\'. Ensure the field exists and is modifiable."' + ): + update_grid_and_label(temp_json_file, new_grid_label, new_grid, new_nom_res) + +def test_missing_grid_label_field(temp_json_file): + """ + Test behavior when the 'grid_label' field is missing in the JSON file. + """ + # Arrange + with open(temp_json_file, "r+", encoding="utf-8") as file: + data = json.load(file) + del data["grid_label"] # Remove the 'grid_label' field + file.seek(0) + json.dump(data, file, indent=4) + file.truncate() + + new_grid_label = "updated_label" + new_grid = "updated_grid" + new_nom_res = "updated_nom_res" + + # Act & Assert + with pytest.raises(KeyError, match="Error while updating 'grid_label'"): + update_grid_and_label(temp_json_file, new_grid_label, new_grid, new_nom_res) + +def test_missing_grid_field(temp_json_file): + """ + Test behavior when the 'grid' field is missing in the JSON file. + """ + # Arrange + with open(temp_json_file, "r+", encoding="utf-8") as file: + data = json.load(file) + del data["grid"] # Remove the 'grid' field + file.seek(0) + json.dump(data, file, indent=4) + file.truncate() + + new_grid_label = "updated_label" + new_grid = "updated_grid" + new_nom_res = "updated_nom_res" + + # Act & Assert + with pytest.raises(KeyError, match="Error while updating 'grid'"): + update_grid_and_label(temp_json_file, new_grid_label, new_grid, new_nom_res) + +def test_invalid_json_file(tmp_path): + """ + Test behavior when the input file is not a valid JSON file. + """ + # Arrange + invalid_json_file = tmp_path / "invalid_file.json" + with open(invalid_json_file, "w", encoding="utf-8") as file: + file.write("This is not a valid JSON!") + + new_grid_label = "updated_label" + new_grid = "updated_grid" + new_nom_res = "updated_nom_res" + + # Act & Assert + with pytest.raises(json.JSONDecodeError): + update_grid_and_label(invalid_json_file, new_grid_label, new_grid, new_nom_res) + +def test_nonexistent_file(): + """ + Test behavior when the specified JSON file does not exist. + """ + # Arrange + nonexistent_file = Path("nonexistent.json") + new_grid_label = "updated_label" + new_grid = "updated_grid" + new_nom_res = "updated_nom_res" + + # Act & Assert + with pytest.raises(FileNotFoundError): + update_grid_and_label(nonexistent_file, new_grid_label, new_grid, new_nom_res) diff --git a/fremorizer/tests/test_cmor_mixer_calendar_integration.py b/fremorizer/tests/test_cmor_mixer_calendar_integration.py new file mode 100644 index 0000000..18b12cc --- /dev/null +++ b/fremorizer/tests/test_cmor_mixer_calendar_integration.py @@ -0,0 +1,146 @@ +""" +Calendar-type integration tests for cmor_run_subtool +===================================================== + +These tests verify that cmor_run_subtool correctly handles the +``calendar_type`` parameter: + +- When a calendar_type is provided → ``update_calendar_type`` is called. +- When calendar_type is None → ``update_calendar_type`` is NOT called. + +Heavy internal work (file globbing, CMORization, outpath updating) is +mocked so the tests focus exclusively on the calendar-type routing logic. +""" + +import json +from unittest.mock import patch + +import pytest + +from fremorizer.cmor_mixer import cmor_run_subtool + + +# --------------------------------------------------------------------------- +# shared fixtures – minimal JSON configs written to tmp_path +# --------------------------------------------------------------------------- + +@pytest.fixture +def exp_config_file(tmp_path): + """Minimal CMIP6 experiment configuration JSON (with outpath for update_outpath).""" + config = { + "mip_era": "cmip6", + "calendar": "360_day", + "grid": "test_grid", + "grid_label": "gn", + "nominal_resolution": "1 km", + "outpath": "." + } + path = tmp_path / "exp_config.json" + path.write_text(json.dumps(config)) + return str(path) + + +@pytest.fixture +def var_list_file(tmp_path): + """Variable-list JSON mapping local names to target names.""" + path = tmp_path / "var_list.json" + path.write_text(json.dumps({"temp": "temp", "salt": "salt"})) + return str(path) + + +@pytest.fixture +def table_config_file(tmp_path): + """MIP-table JSON with two stub variable entries.""" + table = { + "variable_entry": { + "temp": {"frequency": "mon"}, + "salt": {"frequency": "mon"} + } + } + path = tmp_path / "table_config.json" + path.write_text(json.dumps(table)) + return str(path) + + +@pytest.fixture +def fake_nc_filenames(tmp_path): + """Return a list of fake NetCDF filenames that glob would find.""" + return [ + str(tmp_path / "mock_test_file.00010101-00041231.temp.nc"), + str(tmp_path / "mock_test_file.00010101-00041231.salt.nc") + ] + + +# --------------------------------------------------------------------------- +# tests +# --------------------------------------------------------------------------- + +# Shared set of mocks – everything that would touch the filesystem or CMOR +_MOCKS = [ + 'fremorizer.cmor_mixer.update_calendar_type', + 'fremorizer.cmor_mixer.cmorize_all_variables_in_dir', + 'fremorizer.cmor_mixer.glob.glob', +] + + +@patch(_MOCKS[0]) +@patch(_MOCKS[1]) +@patch(_MOCKS[2]) +def test_cmor_run_w_cal_type( + mock_glob, + mock_cmorize, + mock_update_cal, + exp_config_file, + var_list_file, + table_config_file, + fake_nc_filenames, + tmp_path +): + """When calendar_type is provided, update_calendar_type must be called.""" + + mock_glob.return_value = fake_nc_filenames + mock_cmorize.return_value = 0 + calendar_type = "noleap" + + cmor_run_subtool( + indir = str(tmp_path), + json_var_list = var_list_file, + json_table_config = table_config_file, + json_exp_config = exp_config_file, + outdir = str(tmp_path / "output"), + calendar_type = calendar_type + ) + + mock_update_cal.assert_called_once_with( + exp_config_file, calendar_type, output_file_path=None + ) + + +@patch(_MOCKS[0]) +@patch(_MOCKS[1]) +@patch(_MOCKS[2]) +def test_cmor_run_no_cal_type( + mock_glob, + mock_cmorize, + mock_update_cal, + exp_config_file, + var_list_file, + table_config_file, + fake_nc_filenames, + tmp_path +): + """When calendar_type is None (default), update_calendar_type must NOT be called.""" + + mock_glob.return_value = fake_nc_filenames + mock_cmorize.return_value = 0 + + cmor_run_subtool( + indir = str(tmp_path), + json_var_list = var_list_file, + json_table_config = table_config_file, + json_exp_config = exp_config_file, + outdir = str(tmp_path / "output"), + calendar_type = None + ) + + mock_update_cal.assert_not_called() diff --git a/fremorizer/tests/test_cmor_run_subtool.py b/fremorizer/tests/test_cmor_run_subtool.py new file mode 100644 index 0000000..d6e2fd1 --- /dev/null +++ b/fremorizer/tests/test_cmor_run_subtool.py @@ -0,0 +1,406 @@ +''' +tests for fremorizer.cmor_run_subtool +''' + +from datetime import date +import json +import os +from pathlib import Path +import subprocess +import shutil + +import pytest + +from fremorizer import cmor_run_subtool + + +# where are we? we're running pytest from the base directory of this repo +ROOTDIR = 'fremorizer/tests/test_files' + +# setup- cmip/cmor variable table(s) +CMIP6_TABLE_REPO_PATH = \ + f'{ROOTDIR}/cmip6-cmor-tables' +TABLE_CONFIG = \ + f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_Omon.json' + +def test_setup_cmor_cmip_table_repo(): + ''' + setup routine, make sure the recursively cloned tables exist + ''' + assert all( [ Path(CMIP6_TABLE_REPO_PATH).exists(), + Path(TABLE_CONFIG).exists() + ] ) + +# explicit inputs to tool +GRID = 'regridded to FOO grid from native' #placeholder value +GRID_LABEL = 'gr' +NOM_RES = '10000 km' #placeholder value + +INDIR = f'{ROOTDIR}/ocean_sos_var_file' +VARLIST = f'{ROOTDIR}/varlist' +EXP_CONFIG = f'{ROOTDIR}/CMOR_input_example.json' +OUTDIR = f'{ROOTDIR}/outdir' +TMPDIR = f'{OUTDIR}/tmp' + +# input file details. if calendar matches data, the dates should be preserved or equiv. +DATETIMES_INPUTFILE='199301-199302' +FILENAME = f'reduced_ocean_monthly_1x1deg.{DATETIMES_INPUTFILE}.sos' +FULL_INPUTFILE=f"{INDIR}/{FILENAME}.nc" +CALENDAR_TYPE = 'julian' + +# determined by cmor_run_subtool +YYYYMMDD = date.today().strftime('%Y%m%d') +CMOR_CREATES_DIR = \ + f'CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1/Omon/sos/{GRID_LABEL}' +FULL_OUTPUTDIR = \ + f"{OUTDIR}/{CMOR_CREATES_DIR}/v{YYYYMMDD}" +FULL_OUTPUTFILE = \ +f"{FULL_OUTPUTDIR}/sos_Omon_PCMDI-test-1-0_piControl-withism_r3i1p1f1_{GRID_LABEL}_{DATETIMES_INPUTFILE}.nc" + + +def test_setup_fre_cmor_run_subtool(capfd): + ''' + The routine generates a netCDF file from an ascii (cdl) file. It also checks for a ncgen + output file from prev pytest runs, removes it if it's present, and ensures the new file is + created without error. + ''' + + ncgen_input = f"{ROOTDIR}/reduced_ascii_files/{FILENAME}.cdl" + ncgen_output = f"{ROOTDIR}/ocean_sos_var_file/{FILENAME}.nc" + + if Path(ncgen_output).exists(): + Path(ncgen_output).unlink() + assert Path(ncgen_input).exists() + + ex = [ 'ncgen3', '-k', 'netCDF-4', '-o', ncgen_output, ncgen_input ] + + sp = subprocess.run(ex, check = True) + + assert all( [ sp.returncode == 0, Path(ncgen_output).exists() ] ) + + if Path(FULL_OUTPUTFILE).exists(): + Path(FULL_OUTPUTFILE).unlink() + + assert not Path(FULL_OUTPUTFILE).exists() + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_case1(capfd): + ''' fre cmor run, test-use case ''' + + #import sys + #assert False, f'{sys.path}' + + #debug + #print( + # f"cmor_run_subtool(" + # f"\'{INDIR}\'," + # f"\'{VARLIST}\'," + # f"\'{TABLE_CONFIG}\'," + # f"\'{EXP_CONFIG}\'," + # f"\'{OUTDIR}\'" + # ")" + #) + + # test call, where meat of the workload gets done + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST, + json_table_config = TABLE_CONFIG, + json_exp_config = EXP_CONFIG, + outdir = OUTDIR, + run_one_mode = True, + grid_label = GRID_LABEL, + grid = GRID, + nom_res = NOM_RES, + calendar_type = CALENDAR_TYPE + ) + + assert all( [ Path(FULL_OUTPUTFILE).exists(), + Path(FULL_INPUTFILE).exists() ] ) + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_case1_output_compare_data(capfd): + ''' I/O data-only comparison of test case1 ''' + print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') + print(f'FULL_INPUTFILE={FULL_INPUTFILE}') + + nccmp_cmd= [ "nccmp", "-f", "-d", + f"{FULL_INPUTFILE}", + f"{FULL_OUTPUTFILE}" ] + print(f"via subprocess, running {' '.join(nccmp_cmd)}") + result = subprocess.run( ' '.join(nccmp_cmd), + shell=True, + check=False, + capture_output=True + ) + + # err_list has length two if end in newline + err_list = result.stderr.decode().split('\n') + expected_err = \ + "DIFFER : FILE FORMATS : NC_FORMAT_NETCDF4 <> NC_FORMAT_NETCDF4_CLASSIC" + assert all( [result.returncode == 1, + len(err_list)==2, + '' in err_list, + expected_err in err_list ] ) + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_case1_output_compare_metadata(capfd): + ''' I/O metadata-only comparison of test case1 ''' + print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') + print(f'FULL_INPUTFILE={FULL_INPUTFILE}') + + nccmp_cmd= [ "nccmp", "-f", "-m", "-g", + f"{FULL_INPUTFILE}", + f"{FULL_OUTPUTFILE}" ] + print(f"via subprocess, running {' '.join(nccmp_cmd)}") + result = subprocess.run( ' '.join(nccmp_cmd), + shell=True, + check=False + ) + + assert result.returncode == 1 + _out, _err = capfd.readouterr() + + +# FYI, but again, helpful for tests +FILENAME_DIFF = \ + f'reduced_ocean_monthly_1x1deg.{DATETIMES_INPUTFILE}.sosV2.nc' +FULL_INPUTFILE_DIFF = \ + f"{INDIR}/{FILENAME_DIFF}" +VARLIST_DIFF = \ + f'{ROOTDIR}/varlist_local_target_vars_differ' +def test_setup_fre_cmor_run_subtool_case2(capfd): + ''' make a copy of the input file to the slightly different name. + checks for outputfile from prev pytest runs, removes it if it's present. + this routine also checks to make sure the desired input file is present''' + if Path(FULL_OUTPUTFILE).exists(): + Path(FULL_OUTPUTFILE).unlink() + assert not Path(FULL_OUTPUTFILE).exists() + + if Path(OUTDIR+'/CMIP6').exists(): + shutil.rmtree(OUTDIR+'/CMIP6') + assert not Path(OUTDIR+'/CMIP6').exists() + + + # VERY ANNOYING !!! FYI WARNING TODO + if Path(TMPDIR).exists(): + try: + shutil.rmtree(TMPDIR) + except OSError as exc: + print(f'WARNING: TMPDIR={TMPDIR} could not be removed.') + print( ' this does not matter that much, but is unfortunate.') + print( ' suspicion: something the cmor module is using is not being closed') + print(f' exc = {exc}') + + #assert not Path(TMPDIR).exists() # VERY ANNOYING !!! FYI WARNING TODO + + # VERY ANNOYING !!! FYI WARNING TODO + if Path(OUTDIR).exists(): + try: + shutil.rmtree(OUTDIR) + except OSError as exc: + print(f'WARNING: OUTDIR={OUTDIR} could not be removed.') + print( ' this does not matter that much, but is unfortunate.') + print( ' suspicion: something the cmor module is using is not being closed') + print(f' exc = {exc}') + + #assert not Path(OUTDIR).exists() # VERY ANNOYING !!! FYI WARNING TODO + + # make a copy of the usual test file. + if not Path(FULL_INPUTFILE_DIFF).exists(): + shutil.copy( + Path(FULL_INPUTFILE), + Path(FULL_INPUTFILE_DIFF) ) + assert Path(FULL_INPUTFILE_DIFF).exists() + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_case2(capfd): + ''' fre cmor run, test-use case2 ''' + + #debug + #print( + # f"cmor_run_subtool(" + # f"\'{INDIR}\'," + # f"\'{VARLIST_DIFF}\'," + # f"\'{TABLE_CONFIG}\'," + # f"\'{EXP_CONFIG}\'," + # f"\'{OUTDIR}\'" + # ")" + #) + + # test call, where meat of the workload gets done + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST_DIFF, + json_table_config = TABLE_CONFIG, + json_exp_config = EXP_CONFIG, + outdir = OUTDIR, + run_one_mode = True, + grid_label = GRID_LABEL, + grid = GRID, + nom_res = NOM_RES, + calendar_type = CALENDAR_TYPE + ) + + # check we ran on the right input file. + assert all( [ Path(FULL_OUTPUTFILE).exists(), + Path(FULL_INPUTFILE_DIFF).exists() ] ) + _out, _err = capfd.readouterr() + + +def test_fre_cmor_run_subtool_case2_output_compare_data(capfd): + ''' I/O data-only comparison of test case2 ''' + print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') + print(f'FULL_INPUTFILE_DIFF={FULL_INPUTFILE_DIFF}') + + nccmp_cmd= [ "nccmp", "-f", "-d", + f"{FULL_INPUTFILE_DIFF}", + f"{FULL_OUTPUTFILE}" ] + print(f"via subprocess, running {' '.join(nccmp_cmd)}") + result = subprocess.run( ' '.join(nccmp_cmd), + shell=True, + check=False, + capture_output=True + ) + + err_list = result.stderr.decode().split('\n')#length two if end in newline + expected_err="DIFFER : FILE FORMATS : NC_FORMAT_NETCDF4 <> NC_FORMAT_NETCDF4_CLASSIC" + assert all( [result.returncode == 1, + len(err_list)==2, + '' in err_list, + expected_err in err_list ] ) + _out, _err = capfd.readouterr() + +def test_fre_cmor_run_subtool_case2_output_compare_metadata(capfd): + ''' I/O metadata-only comparison of test case2 ''' + print(f'FULL_OUTPUTFILE={FULL_OUTPUTFILE}') + print(f'FULL_INPUTFILE_DIFF={FULL_INPUTFILE_DIFF}') + + nccmp_cmd= [ "nccmp", "-f", "-m", "-g", + f"{FULL_INPUTFILE_DIFF}", + f"{FULL_OUTPUTFILE}" ] + print(f"via subprocess, running {' '.join(nccmp_cmd)}") + result = subprocess.run( ' '.join(nccmp_cmd), + shell=True, + check=False + ) + + assert result.returncode == 1 + _out, _err = capfd.readouterr() + +def test_git_cleanup(): + ''' + Performs a git restore on EXP_CONFIG to avoid false positives from + git's record of changed files. It's supposed to change as part of the test. + ''' + is_ci = os.environ.get("GITHUB_WORKSPACE") is not None + if not is_ci: + git_cmd = f"git restore {EXP_CONFIG}" + restore = subprocess.run(git_cmd, + shell=True, + check=False) + check_cmd = f"git status | grep {EXP_CONFIG}" + check = subprocess.run(check_cmd, + shell = True, + check = False) + #first command completed, second found no file in git status + assert all([restore.returncode == 0, + check.returncode == 1]) + +def test_cmor_run_subtool_raise_value_error(): + ''' + test that ValueError raised when required args are absent + ''' + with pytest.raises(ValueError): + cmor_run_subtool( indir = None, + json_var_list = None, + json_table_config = None, + json_exp_config = None, + outdir = None ) + +def test_fre_cmor_run_subtool_no_exp_config(): + ''' + fre cmor run, exception, json_exp_config DNE + ''' + + # test call, where meat of the workload gets done + with pytest.raises(FileNotFoundError): + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST_DIFF, + json_table_config = TABLE_CONFIG, + json_exp_config = 'DOES NOT EXIST', + outdir = OUTDIR + ) + +VARLIST_EMPTY = \ + f'{ROOTDIR}/empty_varlist' +def test_fre_cmor_run_subtool_empty_varlist(): + ''' + fre cmor run, exception, variable list is empty + ''' + + # test call, where meat of the workload gets done + with pytest.raises(ValueError): + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST_EMPTY, + json_table_config = TABLE_CONFIG, + json_exp_config = EXP_CONFIG, + outdir = OUTDIR + ) + + +def test_fre_cmor_run_subtool_opt_var_name_not_in_table(): + ''' fre cmor run, exception, ''' + + # test call, where meat of the workload gets done + with pytest.raises(ValueError): + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST, + json_table_config = TABLE_CONFIG, + json_exp_config = EXP_CONFIG, + outdir = OUTDIR, + opt_var_name="difmxybo" + ) + + +def test_fre_cmor_run_subtool_missing_mip_era(tmp_path): + ''' + KeyError when the exp config JSON has no mip_era entry. + ''' + # create a minimal exp config that is missing 'mip_era' + bad_exp = tmp_path / 'no_mip_era.json' + exp_data = {"institution_id": "TEST", "source_id": "TEST-1-0"} + bad_exp.write_text(json.dumps(exp_data)) + + with pytest.raises(KeyError, match='noncompliant'): + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST, + json_table_config = TABLE_CONFIG, + json_exp_config = str(bad_exp), + outdir = OUTDIR, + ) + + +def test_fre_cmor_run_subtool_unsupported_mip_era(tmp_path): + ''' + ValueError when mip_era is present but not CMIP6 or CMIP7. + ''' + # create an exp config with an unsupported mip_era value + bad_exp = tmp_path / 'bad_mip_era.json' + with open(EXP_CONFIG, 'r', encoding='utf-8') as f: + exp_data = json.load(f) + exp_data['mip_era'] = 'CMIP99' + bad_exp.write_text(json.dumps(exp_data)) + + with pytest.raises(ValueError, match='only supports CMIP6 and CMIP7'): + cmor_run_subtool( + indir = INDIR, + json_var_list = VARLIST, + json_table_config = TABLE_CONFIG, + json_exp_config = str(bad_exp), + outdir = OUTDIR, + ) diff --git a/fremorizer/tests/test_cmor_run_subtool_further_examples.py b/fremorizer/tests/test_cmor_run_subtool_further_examples.py new file mode 100644 index 0000000..73467fb --- /dev/null +++ b/fremorizer/tests/test_cmor_run_subtool_further_examples.py @@ -0,0 +1,203 @@ +''' +expanded set of tests for fremor run focus on cases beyond test_cmor_run_subtool.py +''' + +from datetime import date +from pathlib import Path +import shutil +import glob +#import time +#import platform +import subprocess +import os + +import pytest + +from fremorizer import cmor_run_subtool + + +# global consts for these tests, with no/trivial impact on the results +ROOTDIR='fremorizer/tests/test_files' +CMORBITE_VARLIST=f'{ROOTDIR}/CMORbite_var_list.json' + +# cmip6 variable table(s) +CMIP6_TABLE_REPO_PATH = \ + f'{ROOTDIR}/cmip6-cmor-tables' + +# outputs +OUTDIR = f'{ROOTDIR}/outdir_ppan_only' +TMPDIR = f'{OUTDIR}/tmp' + +# determined by cmor_run_subtool +YYYYMMDD = date.today().strftime('%Y%m%d') +CMOR_CREATES_DIR_BASE = \ + 'CMIP6/CMIP6/ISMIP6/PCMDI/PCMDI-test-1-0/piControl-withism/r3i1p1f1' + +# this file exists basically for users to specify their own information to append to the netcdf file +# i.e., it fills in FOO/BAR/BAZ style values, and what they are currently is totally irrelevant +EXP_CONFIG_DEFAULT=f'{ROOTDIR}/CMOR_input_example.json' # this likely is not sufficient + +CLEANUP_AFTER_EVERY_TEST = False + +def _cleanup(): + # clean up from previous tests + #time.sleep(60) # busy disk issue? possible non-closing netcdf file problem in code + print(OUTDIR) + if Path(f'{OUTDIR}').exists(): + try: + shutil.rmtree(f'{OUTDIR}') + except: + #time.sleep(60) + shutil.rmtree(f'{OUTDIR}') + assert not Path(f'{OUTDIR}').exists() + +MOCK_ARCHIVE_ROOT='fremorizer/tests/test_files/ascii_files/mock_archive' +ESM4_DECK_PP_DIR='cm6/ESM4/DECK/ESM4_historical_D1/gfdl.ncrc4-intel16-prod-openmp/pp' +ESM4_DEV_PP_DIR='USER/CMIP7/ESM4/DEV/ESM4.5v01_om5b04_piC/gfdl.ncrc5-intel23-prod-openmp/pp' +@pytest.mark.parametrize( "testfile_dir,table,opt_var_name,grid_label,start,calendar", + [ + pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/atmos_plev39_cmip/ts/monthly/5yr/zonavg/', + 'AERmonZ', 'ta', 'gr1','1850','noleap', id='AERmonZ_ta_gr1' ), + pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/atmos_scalar/ts/monthly/5yr/', + 'Amon', 'ch4global', 'gr', '1850','noleap', id='Amon_ch4global_gr' ), + pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/LUmip_refined/ts/monthly/5yr/', + 'Emon', 'gppLut', 'gr1','1850','noleap', id='Emon_gppLut_gr1' ), + pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/atmos_level_cmip/ts/monthly/5yr/', + 'Amon', 'cl', 'gr1','1850','noleap', id='Amon_cl_gr1' ), + pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DECK_PP_DIR}/atmos_level_cmip/ts/monthly/5yr/', + 'Amon', 'mc', 'gr1','1850','noleap', id='Amon_mc_gr1' ), + pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}/ocean_monthly_z_1x1deg/ts/monthly/5yr/', + 'Omon', 'so', 'gr', '0001','noleap', id='Omon_so_gr' ), + pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}/ocean_monthly/ts/monthly/5yr/', + 'Omon', 'sos', 'gn', '0001','noleap', id='Omon_sos_gn' ), + pytest.param(f'{MOCK_ARCHIVE_ROOT}/{ESM4_DEV_PP_DIR}/land/ts/monthly/5yr/', + 'Lmon', 'lai', 'gr1','0001','noleap', id='Lmon_lai_gr1' ), + ] ) + +def test_case_function(testfile_dir,table,opt_var_name,grid_label,start,calendar,monkeypatch): + ''' + Should be iterating over the test dictionary + ''' + + # for native-grid ocean tests, prevent the gold statics lookup from finding + # /archive files so the test uses its own locally-generated statics file + if grid_label == 'gn': + monkeypatch.setattr( + 'fremorizer.cmor_mixer.find_gold_ocean_statics_file', lambda **kw: None) + + # define inputs to the cmor run tool + indir = testfile_dir + table_file = f'{CMIP6_TABLE_REPO_PATH}/Tables/CMIP6_{table}.json' + + # if we can't find the input test file, do an xfail. most likely, you're not at PPAN. + if not Path(indir).exists(): + pytest.xfail(f'{opt_var_name}, {Path(table_file).name}, {grid_label} ' + 'SUCCEEDs on PP/AN at GFDL only! OR testfile_dir does not exist!') + + # execute the test + try: + cdl_input_files=glob.glob(indir+'*.'+opt_var_name+'.cdl') + assert len(cdl_input_files)>=1 + + cdl_input_file=cdl_input_files[0] + assert Path(cdl_input_file).exists() + + nc_input_file=cdl_input_file.replace('.cdl','.nc') + if Path(nc_input_file).exists(): + Path(nc_input_file).unlink() + subprocess.run(['ncgen3','-k','netCDF-4','-o', nc_input_file, cdl_input_file], + check=True) + assert Path(nc_input_file).exists() + + # exception: these files need a ps file to be around, so extra ncgen step for these: + if opt_var_name in [ 'cl', 'mc' ]: + cdl_input_ps_file = cdl_input_file.replace( opt_var_name+'.cdl', 'ps.cdl') + assert Path(cdl_input_ps_file).exists() + + nc_input_ps_file = cdl_input_ps_file.replace('.cdl','.nc') + if Path(nc_input_ps_file).exists(): + Path(nc_input_ps_file).unlink() + subprocess.run(['ncgen3','-k','netCDF-4','-o', nc_input_ps_file, cdl_input_ps_file], + check=True) + assert Path(nc_input_ps_file).exists() + + elif opt_var_name == 'sos': + cdl_ocn_statics_file=testfile_dir.replace('ts/monthly/5yr/','ocean_monthly.static.cdl') + assert Path(cdl_ocn_statics_file).exists() + + nc_ocn_statics_file=cdl_ocn_statics_file.replace('.cdl','.nc') + if Path(nc_ocn_statics_file).exists(): + Path(nc_ocn_statics_file).unlink() + subprocess.run(['ncgen3','-k','netCDF-4','-o', nc_ocn_statics_file, cdl_ocn_statics_file], + check=True) + assert Path(nc_ocn_statics_file).exists() + + ##assert False + ## Debug, please keep. -Ian + #print( + #f'fre -vv cmor run \\\n' + #f' -d {indir} \\\n' + #f' -l {CMORBITE_VARLIST} \\\n' + #f' -r {table_file} \\\n' + #f' -p {EXP_CONFIG_DEFAULT} \\\n' + #f' -o {OUTDIR} \\\n' + #f' --run_one \\\n' + #f' -v {opt_var_name} \\\n' + #f' --grid_desc \'FOO_PLACEHOLDER\' \\\n' + #f' -g {grid_label} \\\n' + #f' --nom_res \'10000 km\' \\\n' + #f' --start {start} \\\n' + #f' --calendar {calendar}\n' + #f'') + #assert False + cmor_run_subtool( + indir = indir, + json_var_list = CMORBITE_VARLIST, + json_table_config = table_file, + json_exp_config = EXP_CONFIG_DEFAULT, + outdir = OUTDIR, + run_one_mode = True, + opt_var_name = opt_var_name, + grid = 'FOO_PLACEHOLDER', + grid_label = grid_label, + nom_res = '10000 km' ,# placeholder + start = start, + calendar_type=calendar + ) + #assert False + some_return = 0 + except Exception as exc: + raise Exception(f'exception caught: exc=\n{exc}') from exc + + # outputs that should be created + cmor_output_dir = f'{OUTDIR}/{CMOR_CREATES_DIR_BASE}/{table}/{opt_var_name}/{grid_label}/v{YYYYMMDD}' + cmor_output_file_glob = f'{cmor_output_dir}/' + \ + f'{opt_var_name}_{table}_PCMDI-test-1-0_piControl-withism_r3i1p1f1_{grid_label}_??????-??????.nc' + cmor_output_file = glob.glob( cmor_output_file_glob )[0] + + # success criteria + assert all( [ some_return == 0, + Path(cmor_output_dir).exists(), + Path(cmor_output_file).exists() ] ) + + if CLEANUP_AFTER_EVERY_TEST: + _cleanup() + +def test_git_cleanup(): + ''' + Performs a git restore on EXP_CONFIG to avoid false positives from + git's record of changed files. It's supposed to change as part of the test. + ''' + is_ci = os.environ.get("GITHUB_WORKSPACE") is not None + if not is_ci: + git_cmd = f"git restore {EXP_CONFIG_DEFAULT}" + restore = subprocess.run(git_cmd, + shell=True, + check=False) + check_cmd = f"git status | grep {EXP_CONFIG_DEFAULT}" + check = subprocess.run(check_cmd, + shell = True, + check = False) + #first command completed, second found no file in git status + assert all( [ restore.returncode == 0, + check.returncode == 1 ] ) diff --git a/fremorizer/tests/test_cmor_yamler.py b/fremorizer/tests/test_cmor_yamler.py new file mode 100644 index 0000000..3def081 --- /dev/null +++ b/fremorizer/tests/test_cmor_yamler.py @@ -0,0 +1,10 @@ +import pytest +from fremorizer.cmor_yamler import cmor_yaml_subtool + +def test_modelyaml_dne_raise_filenotfound(): + with pytest.raises(FileNotFoundError): + cmor_yaml_subtool( yamlfile = "MODEL YAML DOESN'T EXIST", + exp_name = "FOO", + platform = "BAR", + target = "BAZ", + dry_run_mode = True ) diff --git a/fremorizer/tests/test_cmor_yamler_freq_validation.py b/fremorizer/tests/test_cmor_yamler_freq_validation.py new file mode 100644 index 0000000..92616a6 --- /dev/null +++ b/fremorizer/tests/test_cmor_yamler_freq_validation.py @@ -0,0 +1,70 @@ +''' +tests for fremorizer.cmor_helpers.conv_mip_to_bronx_freq +''' + +import pytest + +from fremorizer.cmor_helpers import conv_mip_to_bronx_freq + +def test_conv_mip_to_bronx_freq_valid_frequencies(): + """ + Test conversion of valid MIP table frequencies to bronx frequencies. + """ + # Test cases from the mapping dictionary + test_cases = [ + ("1hr", "1hr"), + ("1hrCM", None), + ("1hrPt", None), + ("3hr", "3hr"), + ("3hrPt", None), + ("6hr", "6hr"), + ("6hrPt", None), + ("day", "daily"), + ("dec", None), + ("fx", None), + ("mon", "monthly"), + ("monC", None), + ("monPt", None), + ("subhrPt", None), + ("yr", "annual"), + ("yrPt", None) # Should return None according to mapping + ] + + for cmor_freq, expected_bronx_freq in test_cases: + result = conv_mip_to_bronx_freq(cmor_freq) + assert result == expected_bronx_freq, f"Failed for {cmor_freq}: expected {expected_bronx_freq}, got {result}" + +def test_conv_mip_to_bronx_freq_invalid_frequency(): + """ + Test that invalid frequencies (not 'fx') raise KeyError. + """ + # Arrange + invalid_frequencies = ["invalid", "unknown", "bad_freq", "yearly", "weekly"] + + for invalid_freq in invalid_frequencies: + # Act & Assert + with pytest.raises(KeyError, match=f'MIP table frequency = "{invalid_freq}" is not a valid MIP frequency'): + conv_mip_to_bronx_freq(invalid_freq) + +def test_conv_mip_to_bronx_freq_edge_cases(): + """ + Test edge cases and boundary conditions. + """ + # Test empty string + with pytest.raises(KeyError): + conv_mip_to_bronx_freq("") + + # Test None input - should raise KeyError + with pytest.raises(KeyError): + conv_mip_to_bronx_freq(None) + +def test_conv_mip_to_bronx_freq_case_sensitivity(): + """ + Test that the function is case-sensitive. + """ + # These should raise KeyError because they're not exact matches + case_variants = ["1HR", "Mon", "DAY", "YR"] + + for variant in case_variants: + with pytest.raises(KeyError): + conv_mip_to_bronx_freq(variant) diff --git a/fremorizer/tests/test_cmor_yamler_subtool.py b/fremorizer/tests/test_cmor_yamler_subtool.py new file mode 100644 index 0000000..3576d8d --- /dev/null +++ b/fremorizer/tests/test_cmor_yamler_subtool.py @@ -0,0 +1,635 @@ +''' +tests for fremorizer.cmor_yamler.cmor_yaml_subtool + +Covers: + - full end-to-end run (dry_run_mode=False) via mocked consolidate_yamls + - every documented exception path in the function +''' + +import json +import shutil +import subprocess +from pathlib import Path +from unittest.mock import patch + +import pytest + +import fremorizer +from fremorizer.cmor_yamler import cmor_yaml_subtool + + +# ---- paths to the existing repo test fixtures ---- +#ROOTDIR = 'fre/tests/test_files' +ROOTDIR = str( Path( fremorizer.__file__ ).parent ) + '/tests/test_files' +CMIP6_TABLE_DIR = f'{ROOTDIR}/cmip6-cmor-tables/Tables' +CMIP6_TABLE_CONFIG = f'{CMIP6_TABLE_DIR}/CMIP6_Omon.json' +VARLIST = f'{ROOTDIR}/varlist' +EXP_CONFIG = f'{ROOTDIR}/CMOR_input_example.json' +CDL_SOURCE = f'{ROOTDIR}/reduced_ascii_files/reduced_ocean_monthly_1x1deg.199301-199302.sos.cdl' +NC_FILENAME = 'reduced_ocean_monthly_1x1deg.199301-199302.sos.nc' + +GRID = 'regridded to FOO grid from native' +GRID_LABEL = 'gr' +NOM_RES = '10000 km' + + +# ---- helpers ---- + +def _build_cmor_dict(*, pp_dir, table_dir, outdir, exp_config, + varlist, mip_era='CMIP6', table_name='Omon', + freq='monthly', component='ocean_monthly_1x1deg', + chunk='P5Y', data_series_type='ts', + gridding=None, start='1993', stop='1993', + calendar_type='julian'): + '''Build the dictionary that consolidate_yamls would return.''' + if gridding is None: + gridding = { + 'grid_label': GRID_LABEL, + 'grid_desc': GRID, + 'nom_res': NOM_RES, + } + return { + 'cmor': { + 'mip_era': mip_era, + 'directories': { + 'pp_dir': pp_dir, + 'table_dir': table_dir, + 'outdir': outdir, + }, + 'exp_json': exp_config, + 'start': start, + 'stop': stop, + 'calendar_type': calendar_type, + 'table_targets': [ + { + 'table_name': table_name, + 'freq': freq, + 'gridding': gridding, + 'target_components': [ + { + 'component_name': component, + 'chunk': chunk, + 'data_series_type': data_series_type, + 'variable_list': varlist, + } + ], + } + ], + } + } + + +@pytest.fixture +def yamler_env(tmp_path): + ''' + Set up a temporary pp directory tree and output directory that + cmor_yaml_subtool can use for a real (non-dry-run) CMIP6 Omon/sos test. + ''' + component = 'ocean_monthly_1x1deg' + freq = 'monthly' + chunk_bronx = '5yr' + + # build pp-like tree: pp_dir / component / ts / monthly / 5yr + indir = tmp_path / 'pp' / component / 'ts' / freq / chunk_bronx + indir.mkdir(parents=True) + + # generate the .nc from the CDL + nc_target = indir / NC_FILENAME + subprocess.run( + ['ncgen3', '-k', 'netCDF-4', '-o', str(nc_target), CDL_SOURCE], + check=True) + assert nc_target.exists() + + outdir = tmp_path / 'cmor_output' + outdir.mkdir() + + # copy exp_config so CMOR can mutate it without touching the repo copy + local_exp_config = tmp_path / 'exp_config.json' + shutil.copy(EXP_CONFIG, local_exp_config) + + # create a dummy yamlfile so check_path_existence passes + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + + return { + 'pp_dir': str(tmp_path / 'pp'), + 'outdir': str(outdir), + 'exp_config': str(local_exp_config), + 'table_dir': CMIP6_TABLE_DIR, + 'varlist': str(Path(VARLIST).resolve()), + 'yamlfile': str(dummy_yaml), + 'component': component, + 'freq': freq, + } + + +# ================================================================ +# end-to-end: dry_run_mode=False +# ================================================================ + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_cmor_yaml_subtool_dry_run_false(mock_consolidate, yamler_env): + ''' + Full end-to-end: cmor_yaml_subtool with dry_run_mode=False should + call cmor_run_subtool and produce at least one CMOR-ised .nc file. + ''' + mock_consolidate.return_value = _build_cmor_dict( pp_dir=yamler_env['pp_dir'], + table_dir=yamler_env['table_dir'], + outdir=yamler_env['outdir'], + exp_config=yamler_env['exp_config'], + varlist=yamler_env['varlist'], ) + + cmor_yaml_subtool( + yamlfile=yamler_env['yamlfile'], + exp_name='test', + platform='test', + target='test', + dry_run_mode=False, + run_one_mode=True, + ) + + #print( Path(yamler_env['outdir']) ) + #print( Path(yamler_env['outdir']).rglob('*.nc') ) + #print( list(Path(yamler_env['outdir']).rglob('*.nc'))[0] ) + output_nc_files = list(Path(yamler_env['outdir']).rglob('*.nc')) + assert len(output_nc_files) > 0, \ + 'cmor_yaml_subtool with dry_run=False produced no output' + #assert False + + +# ================================================================ +# exception tests +# ================================================================ + +def test_yamlfile_does_not_exist(): + ''' FileNotFoundError when yamlfile path does not exist ''' + with pytest.raises(FileNotFoundError): + cmor_yaml_subtool( + yamlfile='DOES_NOT_EXIST.yaml', + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_pp_dir_does_not_exist(mock_consolidate, tmp_path): + ''' FileNotFoundError when pp_dir does not exist ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + outdir = tmp_path / 'out' + outdir.mkdir() + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir='/no/such/pp_dir', + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + ) + + with pytest.raises(FileNotFoundError, match='does not exist'): + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_table_dir_does_not_exist(mock_consolidate, tmp_path): + ''' FileNotFoundError when cmip_cmor_table_dir does not exist ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir='/no/such/table_dir', + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + ) + + with pytest.raises(FileNotFoundError, match='does not exist'): + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_exp_json_does_not_exist(mock_consolidate, tmp_path): + ''' FileNotFoundError when exp_json path does not exist ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config='/no/such/exp.json', + varlist=VARLIST, + ) + + with pytest.raises(FileNotFoundError, match='does not exist'): + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_mip_table_file_does_not_exist(mock_consolidate, tmp_path): + ''' FileNotFoundError when the derived json_mip_table_config does not exist ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + # table_dir exists but references a table_name that has no JSON file + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + table_name='NoSuchTable', + ) + + with pytest.raises(FileNotFoundError, match='does not exist'): + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_cmip7_freq_none_raises(mock_consolidate, tmp_path): + ''' ValueError when mip_era=CMIP7 and freq is None ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + # need a table_dir that has a CMIP7_Omon.json — use the cmip7 tables + cmip7_table_dir = f'{ROOTDIR}/cmip7-cmor-tables/tables' + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=cmip7_table_dir, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + mip_era='CMIP7', + table_name='ocean', + freq=None, + ) + + with pytest.raises(ValueError, match='freq is required for CMIP7'): + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_cmip6_freq_none_no_derivation_raises(mock_consolidate, tmp_path): + ''' + ValueError when mip_era=CMIP6, freq is None, and the MIP table + frequency cannot be derived (e.g. fx table). + ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + # Create a fake MIP table whose variable_entry has an unresolvable freq + fake_table_dir = tmp_path / 'tables' + fake_table_dir.mkdir() + fake_table = fake_table_dir / 'CMIP6_FakeFx.json' + fake_table.write_text(json.dumps({ + 'variable_entry': { + 'areacella': {'frequency': 'fx'} + } + })) + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=str(fake_table_dir), + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + mip_era='CMIP6', + table_name='FakeFx', + freq=None, + ) + + with pytest.raises(ValueError, match='not enough frequency information'): + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_cmip6_freq_none_derivation_exception_caught(mock_consolidate, tmp_path): + ''' + When mip_era=CMIP6, freq is None, and get_bronx_freq_from_mip_table + raises a KeyError (e.g. the MIP table JSON has no variable_entry key), + the except (KeyError, TypeError) branch catches it, sets freq = None, + and the subsequent check raises ValueError. + Covers the except branch around get_bronx_freq_from_mip_table. + ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + # Create a MIP table that is missing the 'variable_entry' key entirely, + # so get_bronx_freq_from_mip_table raises KeyError + fake_table_dir = tmp_path / 'tables' + fake_table_dir.mkdir() + fake_table = fake_table_dir / 'CMIP6_FakeBad.json' + fake_table.write_text(json.dumps({ + 'Header': {'table_id': 'Table FakeBad'} + })) + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=str(fake_table_dir), + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + mip_era='CMIP6', + table_name='FakeBad', + freq=None, + ) + + with pytest.raises(ValueError, match='not enough frequency information'): + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_gridding_dict_has_none_value_raises(mock_consolidate, tmp_path): + ''' ValueError when a gridding field is None ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + gridding={ + 'grid_label': GRID_LABEL, + 'grid_desc': None, # <-- triggers the ValueError + 'nom_res': NOM_RES, + }, + ) + + with pytest.raises(ValueError, match='must have all three fields'): + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_outdir_creation_when_missing(mock_consolidate, tmp_path): + ''' + When cmorized_outdir does not exist, the function should create it + (rather than raising). Verify with a dry-run so we only test the + path-creation logic without running CMOR. + ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'brand_new_outdir' # does NOT exist yet + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + ) + + # dry_run_mode=True so we never hit cmor_run_subtool + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + assert outdir.is_dir() + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_outdir_creation_failure_raises_oserror(mock_consolidate, tmp_path): + ''' + OSError when cmorized_outdir does not exist and Path.mkdir fails. + Covers the except branch in the outdir-creation block. + ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + # pick a path that does NOT exist so the mkdir branch is entered + outdir = tmp_path / 'impossible_outdir' + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + ) + + # mock Path.mkdir to raise so the except branch is hit + with patch.object(Path, 'mkdir', side_effect=PermissionError('no permission')): + with pytest.raises(OSError, match='could not create cmorized_outdir'): + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_start_stop_calendar_missing_from_yaml(mock_consolidate, tmp_path): + ''' + When start, stop, and calendar_type are None on the CLI AND absent + from the YAML dict, the function should log warnings and continue + (dry-run mode). Covers the KeyError branches for start/stop/calendar_type. + ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + cmor_dict = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + ) + # remove the keys so the KeyError branches fire + del cmor_dict['cmor']['start'] + del cmor_dict['cmor']['stop'] + del cmor_dict['cmor']['calendar_type'] + + mock_consolidate.return_value = cmor_dict + + # should not raise — the warnings are logged, dry-run continues + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True, + start=None, + stop=None, + calendar_type=None) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_cmip6_freq_none_derivation_succeeds(mock_consolidate, tmp_path): + ''' + When mip_era=CMIP6 and freq is None, but the MIP table carries a + derivable frequency (e.g. Omon → "mon" → "monthly"), the function + should successfully derive freq and continue (dry-run mode). + Covers the successful get_bronx_freq_from_mip_table path. + ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + mip_era='CMIP6', + table_name='Omon', + freq=None, # force derivation from the MIP table + ) + + # Omon has frequency "mon" → bronx "monthly"; should not raise + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True) + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_dry_run_prints_cli_call(mock_consolidate, tmp_path): + ''' + dry_run_mode=True with print_cli_call=True should log the CLI + invocation and never call cmor_run_subtool. + ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + ) + + # should not raise — just log the dry-run CLI call + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True, + print_cli_call=True) + + # no output files should have been created + output_nc = list(Path(str(outdir)).rglob('*.nc')) + assert len(output_nc) == 0 + + +@patch('fremorizer.cmor_yamler.consolidate_yamls') +def test_dry_run_prints_python_call(mock_consolidate, tmp_path): + ''' + dry_run_mode=True with print_cli_call=False should log the Python + cmor_run_subtool(...) invocation. + ''' + dummy_yaml = tmp_path / 'model.yaml' + dummy_yaml.write_text('placeholder') + local_exp = tmp_path / 'exp.json' + shutil.copy(EXP_CONFIG, local_exp) + pp_dir = tmp_path / 'pp' + pp_dir.mkdir() + outdir = tmp_path / 'out' + outdir.mkdir() + + mock_consolidate.return_value = _build_cmor_dict( + pp_dir=str(pp_dir), + table_dir=CMIP6_TABLE_DIR, + outdir=str(outdir), + exp_config=str(local_exp), + varlist=VARLIST, + ) + + cmor_yaml_subtool( + yamlfile=str(dummy_yaml), + exp_name='x', platform='x', target='x', + dry_run_mode=True, + print_cli_call=False) + + output_nc = list(Path(str(outdir)).rglob('*.nc')) + assert len(output_nc) == 0 diff --git a/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json b/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json new file mode 100644 index 0000000..244bd67 --- /dev/null +++ b/fremorizer/tests/test_files/CMOR_CMIP7_input_example.json @@ -0,0 +1,63 @@ +{ + "#_TESTING_ONLY": " ***** This is for unit-test functionality of NOAA-GDFL's fre.cmor submodule for CMIP7, they do not reflect values used in actual production *****", + "contact ": "MIP participant mipmember@foobar.c.om", + "comment": "additional important information not fitting into other fields can be placed here", + "#_TRACKING": "***** anything to do with citing this data, accredation, licensing, and references go here *****", + "license": "CC-BY-4-0; CMIP7 data produced by CCCma is licensed under a Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0). Consult [TODO terms of use link] for terms of use governing CMIP7 output, including citation requirements and proper acknowledgment. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", + "references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html. The ssp245 simulation is described in Dorkey et al. '(Clim. Dyn., 2003, 323-357.)'", + "drs_specs": "MIP-DRS7", + "archive_id": "WCRP", + "license_id": "CC-BY-4-0", + "tracking_prefix": "hdl:21.14107", + "#_MIP_DETAILS": "***** anything to do with identifying the specific MIP activity this configuration file is for *****", + "_cmip7_option": 1, + "mip_era": "CMIP7", + "parent_mip_era": "CMIP7", + "activity_id": "CMIP", + "parent_activity_id": "CMIP", + "#_SOURCE_SECTION": "***** anything to do with identifying this experiment, it's relationships to other experiments, the producers, and their institution *****", + "institution": "", + "institution_id": "CCCma", + "source": "CanESM6-MR:", + "source_id": "CanESM6-MR", + "source_type": "AOGCM ISM AER", + "experiment_id": "esm-piControl", + "sub_experiment": "none", + "sub_experiment_id": "none", + "parent_source_id": "CanESM6-MR", + "parent_experiment_id": "esm-piControl-spinup", + "#_INDICIES": "***** changed from ints to strings for CMIP7 *****", + "realization_index": "r3", + "initialization_index": "i1", + "physics_index": "p1", + "forcing_index": "f3", + "run_variant": "3rd realization", + "parent_variant_label": "r3i1p1f3", + "#_TEMPORAL_INFO": "***** anything to do with describing temporal aspects of the experiment *****", + "parent_time_units": "days since 1850-01-01", + "branch_method": "no parent", + "branch_time_in_child": 59400.0, + "branch_time_in_parent": 0.0, + "calendar": "julian", + "#_SPATIAL_INFO": "***** anything to do with describing physical aspects of the experiment *****", + "grid": "FOO_BAR_PLACEHOLD", + "grid_label": "g99", + "frequency": "mon", + "region": "glb", + "nominal_resolution": "10000 km", + "#_HISTORY_METADATA": "history attribute string and template, to create history field for output file", + "history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.", + "_history_template": "%s ;rewrote data to be consistent with for variable found in table .", + "#_OUTPUT_PATHS": "***** pathing/templates for output files *****", + "#_output_template_NOTE": "***** PCMDI/cmor 4e7f1f3d731077b7f65c188edefac924cc3e2779 Test/test_cmor_CMIP7.py L47 *****", + "#_output": "***** Root directory for output (can be either a relative or full path) *****", + "outpath": ".", + "#_output_path_template": "***** Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax *****", + "output_path_template": "", + "#_output_file_template": "***** Template for output filename using tables keys or global attributes, these should follow the relevant data reference syntax *****", + "output_file_template": "[].nc", + "#_INPUT_CONFIG_PATHS": "***** pathing/templates for input configuation files holding controlled vocabularies *****", + "_controlled_vocabulary_file": "../tables-cvs/cmor-cvs.json", + "_AXIS_ENTRY_FILE": "CMIP7_coordinate.json", + "_FORMULA_VAR_FILE": "CMIP7_formula_terms.json" +} diff --git a/fremorizer/tests/test_files/CMOR_input_example.json b/fremorizer/tests/test_files/CMOR_input_example.json new file mode 100644 index 0000000..17e295e --- /dev/null +++ b/fremorizer/tests/test_files/CMOR_input_example.json @@ -0,0 +1,47 @@ +{ + "#note": " **** The following are set correctly for CMIP6 and should not normally need editing", + "source_type": "AOGCM ISM AER", + "experiment_id": "piControl-withism", + "activity_id": "ISMIP6", + "sub_experiment_id": "none", + "realization_index": "3", + "initialization_index": "1", + "physics_index": "1", + "forcing_index": "1", + "run_variant": "3rd realization", + "parent_experiment_id": "no parent", + "parent_activity_id": "no parent", + "parent_source_id": "no parent", + "parent_variant_label": "no parent", + "parent_time_units": "no parent", + "branch_method": "no parent", + "branch_time_in_child": 59400.0, + "branch_time_in_parent": 0.0, + "institution_id": "PCMDI", + "source_id": "PCMDI-test-1-0", + "calendar": "julian", + "grid": "regridded to FOO grid from native", + "grid_label": "gr", + "nominal_resolution": "10000 km", + "license": "CMIP6 model data produced by Lawrence Livermore PCMDI is licensed under a Creative Commons Attribution 4.0 International License (https://creativecommons.org/licenses/by/4.0/). Consult https://pcmdi.llnl.gov/CMIP6/TermsOfUse for terms of use governing CMIP6 output, including citation requirements and proper acknowledgment. Further information about this data, including some limitations, can be found via the further_info_url (recorded as a global attribute in this file) and at https:///pcmdi.llnl.gov/. The data producers and data providers make no warranty, either express or implied, including, but not limited to, warranties of merchantability and fitness for a particular purpose. All liabilities arising from the supply of the information (including any liability arising in negligence) are excluded to the fullest extent permitted by law.", + "#output": "Root directory for output (can be either a relative or full path)", + "outpath": "CMIP6", + "contact ": "Python Coder (coder@a.b.c.com)", + "history": "Output from archivcl_A1.nce/giccm_03_std_2xCO2_2256.", + "comment": "", + "references": "Model described by Koder and Tolkien (J. Geophys. Res., 2001, 576-591). Also see http://www.GICC.su/giccm/doc/index.html. The ssp245 simulation is described in Dorkey et al. '(Clim. Dyn., 2003, 323-357.)'", + "sub_experiment": "none", + "institution": "", + "source": "PCMDI-test 1.0 (1989)", + "_controlled_vocabulary_file": "CMIP6_CV.json", + "_AXIS_ENTRY_FILE": "CMIP6_coordinate.json", + "_FORMULA_VAR_FILE": "CMIP6_formula_terms.json", + "_cmip6_option": "CMIP6", + "mip_era": "CMIP6", + "parent_mip_era": "no parent", + "tracking_prefix": "hdl:21.14100", + "_history_template": "%s ;rewrote data to be consistent with for variable found in table .", + "#output_path_template": "Template for output path directory using tables keys or global attributes, these should follow the relevant data reference syntax", + "output_path_template": "<_member_id>
", + "output_file_template": "
<_member_id>" +} \ No newline at end of file diff --git a/fremorizer/tests/test_files/CMORbite_var_list.json b/fremorizer/tests/test_files/CMORbite_var_list.json new file mode 100644 index 0000000..5b75e54 --- /dev/null +++ b/fremorizer/tests/test_files/CMORbite_var_list.json @@ -0,0 +1,11 @@ +{ + "lai": "lai", + "t_ref": "t_ref", + "cl": "cl", + "mc": "mc", + "ta": "ta", + "sos": "sos", + "so": "so", + "ch4global": "ch4global", + "gppLut": "gppLut" +} diff --git a/fremorizer/tests/test_files/cmip6-cmor-tables b/fremorizer/tests/test_files/cmip6-cmor-tables new file mode 160000 index 0000000..087fe45 --- /dev/null +++ b/fremorizer/tests/test_files/cmip6-cmor-tables @@ -0,0 +1 @@ +Subproject commit 087fe45d21c082e28723e0f930e4266abe91b853 diff --git a/fremorizer/tests/test_files/cmip7-cmor-tables b/fremorizer/tests/test_files/cmip7-cmor-tables new file mode 160000 index 0000000..1a31c4d --- /dev/null +++ b/fremorizer/tests/test_files/cmip7-cmor-tables @@ -0,0 +1 @@ +Subproject commit 1a31c4d48d4706e4f37dd39265736587b4ef9c89 diff --git a/fremorizer/tests/test_files/empty_varlist b/fremorizer/tests/test_files/empty_varlist new file mode 100644 index 0000000..2c63c08 --- /dev/null +++ b/fremorizer/tests/test_files/empty_varlist @@ -0,0 +1,2 @@ +{ +} diff --git a/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ps.cdl b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ps.cdl new file mode 100644 index 0000000..3c41e6d --- /dev/null +++ b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ps.cdl @@ -0,0 +1,120 @@ +netcdf atmos_cmip.ps { +dimensions: + bnds = 2 ; + lat = 10 ; + lon = 10 ; + time = UNLIMITED ; // (1 currently) +variables: + double bnds(bnds) ; + bnds:long_name = "vertex number" ; + double lat(lat) ; + lat:long_name = "latitude" ; + lat:units = "degrees_N" ; + lat:axis = "Y" ; + lat:bounds = "lat_bnds" ; + double lat_bnds(lat, bnds) ; + lat_bnds:long_name = "latitude bounds" ; + lat_bnds:units = "degrees_N" ; + lat_bnds:axis = "Y" ; + double lon(lon) ; + lon:long_name = "longitude" ; + lon:units = "degrees_E" ; + lon:axis = "X" ; + lon:bounds = "lon_bnds" ; + double lon_bnds(lon, bnds) ; + lon_bnds:long_name = "longitude bounds" ; + lon_bnds:units = "degrees_E" ; + lon_bnds:axis = "X" ; + float ps(time, lat, lon) ; + ps:_FillValue = 1.e+20f ; + ps:missing_value = 1.e+20f ; + ps:units = "Pa" ; + ps:long_name = "Surface Air Pressure" ; + ps:cell_methods = "time: mean" ; + ps:cell_measures = "area: area" ; + ps:standard_name = "surface_air_pressure" ; + ps:interp_method = "conserve_order2" ; + double time(time) ; + time:units = "days since 1979-01-01 00:00:00" ; + time:long_name = "time" ; + time:axis = "T" ; + time:calendar_type = "JULIAN" ; + time:calendar = "julian" ; + time:bounds = "time_bnds" ; + double time_bnds(time, bnds) ; + time_bnds:units = "days since 1979-01-01 00:00:00" ; + time_bnds:long_name = "time axis boundaries" ; + +// global attributes: + :title = "c96L65_am5f9d7r0_amip" ; + :associated_files = "area: 20050101.grid_spec.nc" ; + :grid_type = "regular" ; + :grid_tile = "N/A" ; + :code_release_version = "2024.05" ; + :git_hash = "5d306c05d9fe755cab04eedc8fd3de0d3c8355a0" ; + :creationtime = "Thu Jun 26 22:27:29 2025" ; + :hostname = "pp208" ; + :history = "Tue Jul 1 23:48:29 2025: ncks -d lat,10,19 -d lon,10,19 -d time,0,0 atmos_cmip.200501-200512.ps.nc atmos_cmip.ps.nc\n", + "fregrid --standard_dimension --input_mosaic C96_mosaic.nc --input_file 20050101.atmos_month_cmip --interp_method conserve_order2 --remap_file .fregrid_remap_file_288_by_180.nc --nlon 288 --nlat 180 --scalar_field tas,ts,psl,ps,uas,height10m,vas,sfcWind,hurs,height2m,huss,pr,prsn,prc,evspsbl,tauu,tauv,hfls,hfss,rlds,rlus,rsds,rsus,rsdscs,rsuscs,rldscs,rsdt,rsut,rlut,rlutcs,rsutcs,prw,clt,clwvi,clivi,rtmt,ccb,cct,ci,sci,ta_unmsk,ua_unmsk,va_unmsk,hus_unmsk,hur_unmsk,wap_unmsk,zg_unmsk,ap,b,ap_bnds,b_bnds,lev_bnds,utendnogw,utendogw,time_bnds --output_file out.nc" ; + :external_variables = "area" ; + :NCO = "netCDF Operators version 5.2.4 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco, Citation = 10.1016/j.envsoft.2008.03.004)" ; +data: + + bnds = 1, 2 ; + + lat = -79.5, -78.5, -77.5, -76.5, -75.5, -74.5, -73.5, -72.5, -71.5, -70.5 ; + + lat_bnds = + -80, -79, + -79, -78, + -78, -77, + -77, -76, + -76, -75, + -75, -74, + -74, -73, + -73, -72, + -72, -71, + -71, -70 ; + + lon = 13.125, 14.375, 15.625, 16.875, 18.125, 19.375, 20.625, 21.875, + 23.125, 24.375 ; + + lon_bnds = + 12.5, 13.75, + 13.75, 15, + 15, 16.25, + 16.25, 17.5, + 17.5, 18.75, + 18.75, 20, + 20, 21.25, + 21.25, 22.5, + 22.5, 23.75, + 23.75, 25 ; + + ps = + 67847.9, 67458.55, 67115.11, 66772.38, 66440.77, 66162.93, 65888.8, + 65593.45, 65293.23, 65006.54, + 66789.57, 66383.48, 65977.97, 65632.27, 65374.4, 65085.55, 64794.48, + 64546.18, 64306.85, 64072.05, + 65794.71, 65444.7, 65107.8, 64790.14, 64463.39, 64191.8, 63949.74, + 63687.01, 63413.01, 63075.07, + 64688.98, 64443.55, 64179.7, 63914.74, 63673.48, 63415.69, 63151.54, + 62794.23, 62480.69, 62263.7, + 63798.53, 63496.58, 63344.81, 63203.22, 63014.45, 62753.95, 62468.75, + 62362.88, 62199.16, 62083.03, + 63515.2, 63463.61, 63428.72, 63393.71, 63206.12, 63193.48, 63082.28, + 62763.41, 62952.35, 63084.93, + 64913.63, 65123.55, 65464.87, 65532.16, 65917.46, 65988.91, 65975.77, + 65935.88, 65677.37, 65275.5, + 70025.15, 70817.16, 71348.82, 72221.68, 73337.95, 73954.92, 74331.36, + 74508.41, 74387.87, 74351.83, + 82315.85, 82231.86, 82861.79, 84103.49, 84763.34, 86469.48, 87334.98, + 87688.62, 88297.25, 88394.98, + 95432.23, 95202.2, 95110.34, 95021.3, 95720.12, 96707.44, 96973.3, + 96866.34, 97190.98, 97556.36 ; + + time = 9512.5 ; + + time_bnds = + 9497, 9528 ; +} diff --git a/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_masked.case2.cdl b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_masked.case2.cdl new file mode 100644 index 0000000..0635582 --- /dev/null +++ b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_masked.case2.cdl @@ -0,0 +1,845 @@ +netcdf atmos_cmip.ua_masked { +dimensions: + time = UNLIMITED ; // (1 currently) + plev19 = 19 ; + lat = 10 ; + lon = 10 ; + bnds = 2 ; +variables: + double plev19(plev19) ; + plev19:_FillValue = NaN ; + double time(time) ; + time:_FillValue = NaN ; + time:long_name = "time" ; + time:axis = "T" ; + time:calendar_type = "JULIAN" ; + time:bounds = "time_bnds" ; + time:units = "days since 1979-01-01" ; + time:calendar = "julian" ; + double lat(lat) ; + lat:_FillValue = NaN ; + lat:long_name = "latitude" ; + lat:units = "degrees_N" ; + lat:axis = "Y" ; + lat:bounds = "lat_bnds" ; + double lon(lon) ; + lon:_FillValue = NaN ; + lon:long_name = "longitude" ; + lon:units = "degrees_E" ; + lon:axis = "X" ; + lon:bounds = "lon_bnds" ; + float ua_unmsk(time, plev19, lat, lon) ; + ua_unmsk:_FillValue = 1.e+20f ; + ua_unmsk:units = "m s-1" ; + ua_unmsk:long_name = "Eastward Wind" ; + ua_unmsk:cell_methods = "time: mean" ; + ua_unmsk:cell_measures = "area: area" ; + ua_unmsk:standard_name = "eastward_wind" ; + ua_unmsk:interp_method = "conserve_order2" ; + ua_unmsk:pressure_mask = "False" ; + ua_unmsk:missing_value = 1.e+20 ; + float ua(time, plev19, lat, lon) ; + ua_unmsk:_FillValue = 1.e+20f ; + ua_unmsk:units = "m s-1" ; + ua_unmsk:long_name = "Eastward Wind" ; + ua_unmsk:cell_methods = "time: mean" ; + ua_unmsk:cell_measures = "area: area" ; + ua_unmsk:standard_name = "eastward_wind" ; + ua_unmsk:interp_method = "conserve_order2" ; + ua_unmsk:pressure_mask = "True" ; + ua_unmsk:missing_value = 1.e+20 ; + double bnds(bnds) ; + bnds:_FillValue = NaN ; + bnds:long_name = "vertex number" ; + double lat_bnds(lat, bnds) ; + lat_bnds:_FillValue = NaN ; + lat_bnds:long_name = "latitude bounds" ; + double lon_bnds(lon, bnds) ; + lon_bnds:_FillValue = NaN ; + lon_bnds:long_name = "longitude bounds" ; + double time_bnds(time, bnds) ; + time_bnds:_FillValue = NaN ; + time_bnds:long_name = "time axis boundaries" ; + +// global attributes: + :title = "c96L65_am5f9d7r0_amip" ; + :associated_files = "area: 20050101.grid_spec.nc" ; + :grid_type = "regular" ; + :grid_tile = "N/A" ; + :code_release_version = "2024.05" ; + :git_hash = "5d306c05d9fe755cab04eedc8fd3de0d3c8355a0" ; + :creationtime = "Thu Jun 26 22:27:29 2025" ; + :hostname = "pp208" ; + :history = "Tue Jul 1 23:47:57 2025: ncks -d lat,10,19 -d lon,10,19 -d time,0,0 atmos_cmip.200501-200512.ua_unmsk.nc atmos_cmip.ua_unmsk.nc\nfregrid --standard_dimension --input_mosaic C96_mosaic.nc --input_file 20050101.atmos_month_cmip --interp_method conserve_order2 --remap_file .fregrid_remap_file_288_by_180.nc --nlon 288 --nlat 180 --scalar_field tas,ts,psl,ps,uas,height10m,vas,sfcWind,hurs,height2m,huss,pr,prsn,prc,evspsbl,tauu,tauv,hfls,hfss,rlds,rlus,rsds,rsus,rsdscs,rsuscs,rldscs,rsdt,rsut,rlut,rlutcs,rsutcs,prw,clt,clwvi,clivi,rtmt,ccb,cct,ci,sci,ta_unmsk,ua_unmsk,va_unmsk,hus_unmsk,hur_unmsk,wap_unmsk,zg_unmsk,ap,b,ap_bnds,b_bnds,lev_bnds,utendnogw,utendogw,time_bnds --output_file out.nc" ; + :external_variables = "area" ; + :NCO = "netCDF Operators version 5.2.4 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco, Citation = 10.1016/j.envsoft.2008.03.004)" ; +data: + + plev19 = 100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, + 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000, 500, 100 ; + + time = 9512.5 ; + + lat = -79.5, -78.5, -77.5, -76.5, -75.5, -74.5, -73.5, -72.5, -71.5, -70.5 ; + + lon = 13.125, 14.375, 15.625, 16.875, 18.125, 19.375, 20.625, 21.875, + 23.125, 24.375 ; + + ua = + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + -1.765279, -1.833698, -1.684174, -1.374214, -1.259788, -1.473578, + -1.366698, -1.105641, -1.151468, -1.052101, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, -3.65111, -4.398364, -4.396877, -4.207345, -3.572534, + -5.248218, -4.912538, -4.553966, -4.296717, -4.063207, -4.280607, + -4.150732, -3.721014, -3.682456, -3.493971, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + -4.449573, -5.811647, -6.238008, -5.904782, -5.536724, -5.204103, + -5.139505, -6.584569, -7.315174, -6.587632, + -7.759037, -7.923704, -7.657747, -7.342247, -7.189469, -6.957999, + -6.869977, -6.911683, -6.890206, -6.571945, + -5.766199, -5.629292, -5.500316, -5.457587, -5.137841, -4.946566, + -4.747363, -4.453594, -4.333026, -4.211693, + 1.063227, 1.156564, 1.210127, 1.265683, 1.325263, 1.37135, 1.379399, + 1.395321, 1.408047, 1.413615, + 0.8057297, 0.823011, 0.8517545, 0.8839636, 0.8734952, 0.8716547, 0.8792083, + 0.8959213, 0.933274, 0.9308779, + 0.3776965, 0.3511574, 0.3064211, 0.2771505, 0.2723292, 0.26952, 0.2818037, + 0.2319987, 0.165376, 0.1122594, + -0.2686788, -0.3904975, -0.4879307, -0.5713966, -0.6487127, -0.7579689, + -0.879656, -1.019151, -1.160378, -1.290107, + -1.44353, -1.544772, -1.67499, -1.812897, -1.98716, -2.274205, -2.556359, + -2.771371, -2.97445, -3.201544, + -3.11445, -3.268162, -3.425929, -3.644483, -3.970347, -4.194717, -4.426955, + -4.785159, -4.986393, -5.121875, + -4.842557, -4.942987, -5.010421, -5.153152, -5.254729, -5.326348, + -5.648958, -6.018217, -6.451276, -6.772595, + -6.110239, -6.028393, -5.95482, -5.914704, -5.883089, -5.694447, -5.721324, + -5.85588, -6.038979, -6.222065, + -6.000411, -6.046391, -5.883338, -5.710005, -5.577016, -5.360312, + -5.450391, -5.419651, -5.09429, -4.927251, + -4.122381, -4.0073, -3.920688, -3.890532, -3.619308, -3.348911, -3.164834, + -3.032901, -2.999816, -2.981941, + 1.021295, 1.058732, 1.055263, 1.055828, 1.06735, 1.085502, 1.072789, + 1.066372, 1.055719, 1.03737, + 0.5644462, 0.5258871, 0.505933, 0.5047766, 0.4665931, 0.4326979, 0.4054725, + 0.3554713, 0.3202047, 0.2826239, + -0.09909091, -0.1695052, -0.2423104, -0.308107, -0.3618009, -0.4362217, + -0.5038798, -0.5690737, -0.6322247, -0.6912587, + -0.8452172, -0.9907208, -1.12432, -1.275977, -1.407145, -1.514558, + -1.611202, -1.70489, -1.787341, -1.840466, + -1.660944, -1.836267, -2.043581, -2.208486, -2.365314, -2.544751, + -2.701359, -2.791034, -2.877407, -2.950873, + -2.49663, -2.691726, -2.876794, -3.089699, -3.315934, -3.422023, -3.487426, + -3.497974, -3.590583, -3.699841, + -3.409052, -3.454748, -3.579855, -3.723231, -3.777681, -3.849489, + -3.761075, -3.758978, -3.830392, -3.977781, + -4.332948, -4.30478, -4.268668, -4.142052, -4.038466, -3.906201, -3.853595, + -3.97838, -4.061214, -3.96847, + -4.037989, -3.923016, -3.725387, -3.504999, -3.335768, -3.048127, + -2.881483, -2.777315, -2.57696, -2.525698, + -2.675588, -2.521989, -2.357147, -2.217182, -1.92482, -1.59737, -1.311503, + -1.052147, -0.8554193, -0.7043628, + 0.868393, 0.8096983, 0.7342755, 0.6706711, 0.6231919, 0.5861326, 0.5235301, + 0.4832453, 0.4458269, 0.403123, + 0.3249699, 0.203376, 0.09710471, 0.005840228, -0.09959703, -0.186581, + -0.2659973, -0.3638777, -0.4377598, -0.5044919, + -0.4682975, -0.6448929, -0.7865676, -0.9014125, -1.010695, -1.141248, + -1.26495, -1.354475, -1.432743, -1.508723, + -1.302762, -1.487646, -1.667504, -1.870815, -2.038251, -2.165048, + -2.268736, -2.35599, -2.422225, -2.462221, + -1.897165, -2.089815, -2.293899, -2.467101, -2.626223, -2.795213, + -2.938092, -3.009527, -3.077491, -3.088423, + -2.338803, -2.44815, -2.576407, -2.714023, -2.881914, -2.990129, -3.088588, + -3.162073, -3.200975, -3.280721, + -2.629382, -2.617197, -2.644625, -2.714817, -2.728057, -2.764721, + -2.755495, -2.804715, -2.900975, -3.060241, + -2.806314, -2.699975, -2.604603, -2.467385, -2.316504, -2.181494, + -2.104434, -2.068235, -2.089983, -2.092961, + -2.350076, -2.191863, -2.015235, -1.808842, -1.591449, -1.298141, + -1.055869, -0.8611979, -0.6329716, -0.4672858, + -1.417739, -1.253469, -1.085745, -0.9032833, -0.6274295, -0.3408952, + -0.04521877, 0.2827266, 0.5437686, 0.7892594, + 0.08603396, -0.01244956, -0.1302382, -0.230692, -0.3224648, -0.4244072, + -0.5279509, -0.5862728, -0.6336403, -0.6853408, + -0.5083337, -0.6752201, -0.8401579, -1.021567, -1.198536, -1.330774, + -1.454516, -1.586487, -1.691136, -1.77504, + -1.209975, -1.465892, -1.672402, -1.836556, -2.003343, -2.175733, + -2.340954, -2.462788, -2.570125, -2.671328, + -1.835453, -2.040931, -2.250755, -2.488194, -2.693811, -2.848905, + -2.973454, -3.076011, -3.155657, -3.221975, + -2.087997, -2.262499, -2.42809, -2.583258, -2.728339, -2.874663, -3.012026, + -3.105144, -3.219588, -3.292633, + -2.219754, -2.239856, -2.291842, -2.338402, -2.416109, -2.477032, + -2.551712, -2.625169, -2.696625, -2.812562, + -2.214184, -2.120897, -2.02604, -1.978979, -1.914884, -1.861667, -1.797479, + -1.808812, -1.854385, -1.959785, + -2.145867, -1.937978, -1.761547, -1.585332, -1.406883, -1.215052, + -1.064301, -0.9400889, -0.883372, -0.8476374, + -1.6123, -1.449645, -1.277987, -1.08922, -0.8639328, -0.5785105, + -0.3019016, -0.06757384, 0.1769251, 0.3469639, + -0.4829444, -0.3573219, -0.2495783, -0.1331687, 0.03723535, 0.2394148, + 0.4937723, 0.8273596, 1.081453, 1.332125, + -0.8436456, -0.9486177, -1.059377, -1.153291, -1.241334, -1.344937, + -1.442711, -1.494026, -1.531425, -1.569082, + -1.363601, -1.517111, -1.673566, -1.853703, -2.024385, -2.153884, + -2.277348, -2.399244, -2.49864, -2.581057, + -1.900008, -2.121459, -2.298829, -2.446108, -2.600936, -2.75574, -2.91209, + -3.041326, -3.159974, -3.291958, + -2.332246, -2.485257, -2.649413, -2.831298, -2.996419, -3.138222, + -3.260611, -3.390049, -3.512205, -3.606771, + -2.484051, -2.582164, -2.656566, -2.747351, -2.843552, -2.953647, + -3.080211, -3.176885, -3.30392, -3.404524, + -2.524805, -2.482363, -2.478343, -2.466771, -2.482541, -2.493074, + -2.524661, -2.568022, -2.622594, -2.706606, + -2.361249, -2.253306, -2.134065, -2.054415, -1.964403, -1.889408, + -1.791761, -1.750293, -1.718029, -1.732389, + -2.014955, -1.839144, -1.68769, -1.535044, -1.384408, -1.211162, -1.050033, + -0.8907735, -0.7928692, -0.7119153, + -1.236629, -1.093662, -0.9508283, -0.8085872, -0.6442373, -0.4315634, + -0.1945382, -0.002699569, 0.1975364, 0.3391424, + -0.113028, 0.03701114, 0.1509672, 0.244096, 0.3440107, 0.4850455, 0.659207, + 0.8812482, 1.058411, 1.222257, + -1.854996, -1.890166, -1.915619, -1.932104, -1.946811, -1.971547, + -1.988159, -1.97958, -1.964028, -1.948233, + -2.224256, -2.271724, -2.324106, -2.393641, -2.449779, -2.481474, + -2.509733, -2.527462, -2.532758, -2.527077, + -2.520393, -2.60658, -2.663467, -2.701849, -2.744738, -2.780796, -2.817967, + -2.841355, -2.859165, -2.87995, + -2.696981, -2.736231, -2.781672, -2.833747, -2.878667, -2.9107, -2.931204, + -2.952544, -2.963497, -2.964039, + -2.669573, -2.676523, -2.663651, -2.663748, -2.672803, -2.703568, -2.73891, + -2.744817, -2.765156, -2.767472, + -2.476536, -2.429364, -2.405869, -2.377028, -2.36033, -2.332005, -2.306961, + -2.277795, -2.2701, -2.287573, + -2.063781, -1.986485, -1.90999, -1.85632, -1.792651, -1.720523, -1.636689, + -1.607912, -1.594598, -1.61015, + -1.519658, -1.429246, -1.343379, -1.238522, -1.116604, -1.015946, + -0.9358807, -0.8798366, -0.8500427, -0.8273263, + -0.7674065, -0.6452081, -0.5515807, -0.4693587, -0.3846785, -0.3116637, + -0.2242767, -0.1593899, -0.1116388, -0.06896764, + -0.01183324, 0.1091698, 0.2188423, 0.312474, 0.3854575, 0.4438081, + 0.5140151, 0.5824342, 0.6187019, 0.6379458, + -2.684902, -2.610719, -2.535948, -2.460751, -2.387049, -2.321111, + -2.254766, -2.182338, -2.107151, -2.031494, + -2.820069, -2.740268, -2.659949, -2.584899, -2.516798, -2.444206, + -2.371544, -2.295798, -2.221301, -2.142587, + -2.920424, -2.834318, -2.745224, -2.660668, -2.577662, -2.494831, -2.4213, + -2.345643, -2.268257, -2.192751, + -2.919983, -2.838565, -2.757891, -2.677184, -2.599998, -2.519447, + -2.432564, -2.343707, -2.255901, -2.170567, + -2.795936, -2.718457, -2.634572, -2.554108, -2.473378, -2.391261, + -2.312509, -2.233216, -2.158018, -2.077407, + -2.562569, -2.481418, -2.403707, -2.320077, -2.236581, -2.159199, + -2.083839, -2.011575, -1.949767, -1.892647, + -2.250703, -2.162687, -2.078802, -2.00374, -1.924842, -1.843397, -1.762613, + -1.697863, -1.633513, -1.583002, + -1.825546, -1.773501, -1.720045, -1.646263, -1.569409, -1.491826, + -1.415384, -1.344221, -1.289576, -1.235646, + -1.16891, -1.113447, -1.082653, -1.06613, -1.038724, -1.014261, -0.971841, + -0.9325086, -0.8841423, -0.8362257, + -0.4466377, -0.3950162, -0.3640113, -0.3467484, -0.3497896, -0.3435763, + -0.3285452, -0.3093777, -0.3015329, -0.2992152, + -3.47265, -3.349887, -3.225127, -3.099491, -2.975098, -2.858005, -2.742823, + -2.620348, -2.497191, -2.375009, + -3.669024, -3.520918, -3.372771, -3.233594, -3.098513, -2.952549, + -2.805939, -2.665119, -2.521485, -2.376583, + -3.875113, -3.713779, -3.561024, -3.408706, -3.2558, -3.100774, -2.95319, + -2.807744, -2.659667, -2.510706, + -3.950386, -3.795719, -3.641412, -3.475792, -3.323338, -3.181724, + -3.039761, -2.904638, -2.775108, -2.650002, + -3.947389, -3.783504, -3.606038, -3.452417, -3.30665, -3.171729, -3.049297, + -2.93359, -2.822966, -2.726708, + -3.906445, -3.746139, -3.594896, -3.442577, -3.297046, -3.1648, -3.036906, + -2.916452, -2.812173, -2.712864, + -3.83857, -3.71836, -3.602963, -3.481781, -3.346611, -3.205612, -3.061509, + -2.923938, -2.794568, -2.691266, + -3.559861, -3.530004, -3.478284, -3.396861, -3.302307, -3.188716, + -3.069149, -2.94336, -2.817003, -2.663864, + -2.915129, -2.919575, -2.929287, -2.93514, -2.908199, -2.866826, -2.797559, + -2.709057, -2.578434, -2.428774, + -2.010599, -2.042224, -2.077676, -2.104624, -2.122853, -2.120454, + -2.097497, -2.047853, -1.992431, -1.92157, + -4.22112, -4.07559, -3.922843, -3.768222, -3.614719, -3.473573, -3.340809, + -3.200793, -3.061464, -2.926893, + -4.502892, -4.344226, -4.183433, -4.026276, -3.878217, -3.729609, -3.58069, + -3.431968, -3.287462, -3.145941, + -4.631217, -4.473382, -4.315218, -4.164486, -4.013389, -3.85793, -3.710944, + -3.570176, -3.429156, -3.28708, + -4.626989, -4.476557, -4.326241, -4.174338, -4.032887, -3.895635, + -3.756263, -3.619544, -3.485802, -3.356973, + -4.556738, -4.395508, -4.235921, -4.096162, -3.958964, -3.832275, + -3.715655, -3.600988, -3.489401, -3.387974, + -4.468916, -4.302645, -4.15163, -4.001896, -3.865624, -3.743882, -3.626797, + -3.521506, -3.429111, -3.338457, + -4.44977, -4.292527, -4.144361, -4.001679, -3.853956, -3.707088, -3.567008, + -3.438192, -3.317925, -3.220261, + -4.439157, -4.329401, -4.202333, -4.056956, -3.905393, -3.736642, + -3.568599, -3.403389, -3.260043, -3.114572, + -4.203069, -4.12499, -4.031964, -3.926179, -3.793952, -3.63649, -3.470912, + -3.312045, -3.139288, -2.968029, + -3.549062, -3.518938, -3.466148, -3.407319, -3.308811, -3.204609, + -3.090285, -2.963688, -2.832902, -2.702063, + -4.634931, -4.510322, -4.387384, -4.262967, -4.13814, -4.019274, -3.907151, + -3.793168, -3.677498, -3.562162, + -4.812496, -4.685526, -4.557202, -4.432652, -4.315062, -4.196854, + -4.077685, -3.961596, -3.849445, -3.737238, + -4.926235, -4.801914, -4.682349, -4.564368, -4.445381, -4.328474, + -4.217326, -4.105275, -3.991226, -3.87654, + -4.974308, -4.86126, -4.746056, -4.629657, -4.521437, -4.412365, -4.300677, + -4.186651, -4.075935, -3.968378, + -4.938999, -4.831478, -4.722439, -4.619403, -4.516479, -4.414182, + -4.313823, -4.215149, -4.114635, -4.01406, + -4.802555, -4.709178, -4.621257, -4.533046, -4.446228, -4.362994, + -4.278905, -4.198927, -4.114691, -4.029364, + -4.646967, -4.549815, -4.467683, -4.393793, -4.318109, -4.249639, + -4.184724, -4.118573, -4.056467, -3.996501, + -4.553621, -4.461698, -4.370646, -4.279413, -4.197869, -4.108099, + -4.025449, -3.951957, -3.882595, -3.806807, + -4.450593, -4.350024, -4.254059, -4.160119, -4.047391, -3.930037, + -3.814608, -3.705483, -3.598191, -3.498837, + -4.176404, -4.088185, -3.99167, -3.888712, -3.763858, -3.636919, -3.503574, + -3.371032, -3.253147, -3.15005, + -5.305916, -5.228198, -5.149081, -5.066242, -4.979733, -4.891483, + -4.805131, -4.715399, -4.625538, -4.536609, + -5.409827, -5.338651, -5.264568, -5.190886, -5.118539, -5.040178, + -4.958767, -4.879787, -4.798156, -4.714354, + -5.436405, -5.377537, -5.321352, -5.260819, -5.19785, -5.135309, -5.069129, + -4.99902, -4.925947, -4.848398, + -5.425586, -5.371675, -5.315626, -5.258965, -5.205391, -5.147702, + -5.090356, -5.029489, -4.966897, -4.903516, + -5.421552, -5.364684, -5.306411, -5.247279, -5.189979, -5.13738, -5.082496, + -5.023454, -4.961823, -4.898187, + -5.399646, -5.353481, -5.305019, -5.255711, -5.203397, -5.146911, + -5.086962, -5.023842, -4.958927, -4.891607, + -5.324206, -5.285987, -5.253324, -5.216431, -5.17214, -5.119621, -5.066675, + -5.011673, -4.952978, -4.889044, + -5.201594, -5.185238, -5.161349, -5.129875, -5.091774, -5.047886, + -5.002157, -4.954802, -4.903096, -4.845263, + -5.027672, -5.009829, -4.996872, -4.982873, -4.957479, -4.928816, + -4.894786, -4.855186, -4.809749, -4.755376, + -4.800905, -4.790343, -4.778637, -4.76788, -4.754467, -4.730603, -4.70276, + -4.672376, -4.637187, -4.600635, + -5.877733, -5.806942, -5.737911, -5.665781, -5.589697, -5.508978, + -5.428902, -5.346344, -5.262498, -5.177478, + -5.999748, -5.93775, -5.871769, -5.80268, -5.736236, -5.664495, -5.589656, + -5.515277, -5.436458, -5.355067, + -6.105768, -6.050148, -5.99489, -5.934047, -5.869964, -5.807178, -5.740132, + -5.669778, -5.59723, -5.518231, + -6.204219, -6.156556, -6.104576, -6.049919, -5.996562, -5.93629, -5.873829, + -5.805651, -5.734739, -5.662208, + -6.283307, -6.238613, -6.188774, -6.136417, -6.083857, -6.031115, + -5.974864, -5.914575, -5.849953, -5.782243, + -6.327734, -6.29085, -6.250577, -6.207967, -6.160203, -6.109562, -6.055408, + -5.996362, -5.935081, -5.871264, + -6.328987, -6.301887, -6.276907, -6.244695, -6.205217, -6.156336, + -6.106204, -6.054013, -5.999475, -5.941885, + -6.286625, -6.282731, -6.267518, -6.242774, -6.20835, -6.168219, -6.125497, + -6.080946, -6.031807, -5.975547, + -6.193247, -6.187342, -6.183197, -6.175198, -6.155711, -6.131026, -6.09807, + -6.059319, -6.013174, -5.959172, + -6.058994, -6.055746, -6.05086, -6.046446, -6.038727, -6.018096, -5.98981, + -5.954857, -5.918507, -5.881165, + -6.561323, -6.499266, -6.438377, -6.375708, -6.310773, -6.244139, + -6.180099, -6.115896, -6.051152, -5.986147, + -6.845045, -6.786884, -6.725439, -6.661109, -6.601454, -6.540991, + -6.479097, -6.417562, -6.355603, -6.293946, + -7.102453, -7.045627, -6.989235, -6.932842, -6.874574, -6.816972, + -6.758873, -6.701811, -6.644189, -6.583754, + -7.351357, -7.301179, -7.248499, -7.19398, -7.142448, -7.089523, -7.035904, + -6.981055, -6.926404, -6.871471, + -7.598204, -7.549111, -7.497645, -7.447221, -7.397934, -7.350796, + -7.304009, -7.256029, -7.20568, -7.155462, + -7.82113, -7.78062, -7.739799, -7.69751, -7.652907, -7.609156, -7.56383, + -7.517503, -7.472572, -7.428266, + -8.020307, -7.985536, -7.955499, -7.923059, -7.885509, -7.841635, + -7.801394, -7.76416, -7.727195, -7.691062, + -8.203031, -8.185472, -8.161825, -8.130711, -8.093474, -8.056222, + -8.020072, -7.988377, -7.958443, -7.925798, + -8.343966, -8.324842, -8.308795, -8.291328, -8.267859, -8.240988, -8.21365, + -8.186219, -8.156057, -8.126547, + -8.44836, -8.431485, -8.415438, -8.404657, -8.391961, -8.373449, -8.350844, + -8.326425, -8.308137, -8.293455, + -7.418903, -7.380289, -7.343199, -7.30528, -7.266068, -7.225653, -7.186882, + -7.148771, -7.1104, -7.071782, + -7.886793, -7.851552, -7.814201, -7.774897, -7.739104, -7.703277, + -7.666594, -7.630272, -7.594023, -7.558314, + -8.341415, -8.308264, -8.274553, -8.240778, -8.205718, -8.171395, + -8.137189, -8.103981, -8.07057, -8.035443, + -8.796901, -8.767877, -8.73701, -8.704835, -8.674766, -8.644069, -8.61279, + -8.580782, -8.54905, -8.517601, + -9.253543, -9.225341, -9.195226, -9.165483, -9.137247, -9.111868, + -9.086359, -9.059315, -9.030994, -9.003092, + -9.699064, -9.677286, -9.654745, -9.63162, -9.607246, -9.582701, -9.557117, + -9.531214, -9.505648, -9.481256, + -10.12736, -10.10717, -10.09253, -10.07685, -10.05586, -10.03019, + -10.00788, -9.988334, -9.96856, -9.949014, + -10.53594, -10.52901, -10.51833, -10.50091, -10.47864, -10.45729, + -10.43758, -10.42123, -10.40631, -10.38915, + -10.90329, -10.89263, -10.88709, -10.88161, -10.87157, -10.86071, + -10.84752, -10.83391, -10.81815, -10.80181, + -11.248, -11.23781, -11.23062, -11.22926, -11.23034, -11.22393, -11.21365, + -11.20186, -11.19413, -11.18977, + -13.83239, -13.82074, -13.808, -13.79507, -13.78222, -13.76985, -13.75674, + -13.74387, -13.73124, -13.71895, + -14.99369, -14.98229, -14.97097, -14.96008, -14.94822, -14.9366, -14.92516, + -14.9145, -14.90532, -14.89437, + -16.12242, -16.11533, -16.10389, -16.09248, -16.08173, -16.06978, + -16.06084, -16.05067, -16.03975, -16.03102, + -17.2117, -17.20174, -17.1923, -17.18274, -17.17234, -17.16337, -17.15279, + -17.14514, -17.13551, -17.12395, + -18.23167, -18.22563, -18.21396, -18.20369, -18.19536, -18.19025, -18.1865, + -18.17599, -18.1694, -18.16037, + -19.16084, -19.15458, -19.15068, -19.14543, -19.14352, -19.13608, + -19.12984, -19.12499, -19.11406, -19.10741, + -19.96635, -19.96284, -19.96158, -19.96541, -19.9607, -19.95777, -19.95134, + -19.94964, -19.94552, -19.9427, + -20.64782, -20.64862, -20.65172, -20.64795, -20.64328, -20.63709, -20.6386, + -20.63893, -20.64199, -20.63851, + -21.1739, -21.17439, -21.17778, -21.18302, -21.18665, -21.19431, -21.1958, + -21.1981, -21.19873, -21.19386, + -21.59861, -21.59514, -21.59659, -21.60578, -21.61945, -21.62407, + -21.62483, -21.62446, -21.6267, -21.63452 ; + + ua_unmsk = + -0.3975942, -0.2708012, -0.1832284, -0.09187625, 0.001559797, 0.07204625, + 0.0783459, 0.09246785, 0.1097215, 0.1203732, + -0.5745555, -0.5160208, -0.4475087, -0.3612338, -0.3011235, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.151411, -2.133208, -2.145542, -2.198483, -2.335929, -2.401734, + -2.460874, -2.553266, -2.576532, -2.571083, + -2.586592, -2.56783, -2.469594, -2.440526, -2.463234, -2.546937, -2.664745, + -2.762299, -2.825696, -2.793565, + -2.590438, -2.542438, -2.476797, -2.352731, -2.229772, -2.188571, + -2.246655, -2.410638, -2.534509, -2.572949, + -1.956838, -1.938947, -1.808724, -1.627707, -1.515042, -1.380297, + -1.431068, -1.480767, -1.527824, -1.59102, + -0.5315368, -0.5231183, -0.4564048, -0.3944708, -0.1738839, -0.07352839, + 0.01055507, 0.1166717, 0.1312627, 0.2858787, + -0.3975942, -0.2708012, -0.1832284, -0.09187625, 0.001559797, 0.07204625, + 0.0783459, 0.09246785, 0.1097215, 0.1203732, + -0.5745555, -0.5160208, -0.4475087, -0.3612338, -0.3011235, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.151411, -2.133208, -2.145542, -2.198483, -2.335929, -2.401734, + -2.460874, -2.553266, -2.576532, -2.571083, + -2.586592, -2.56783, -2.469594, -2.440526, -2.463234, -2.546937, -2.664745, + -2.76217, -2.823994, -2.793565, + -2.56729, -2.51979, -2.476788, -2.342977, -2.180678, -2.173315, -2.202794, + -2.382988, -2.51635, -2.56469, + -2.164203, -2.002625, -1.769979, -1.522183, -1.529329, -1.788184, + -1.570996, -1.601109, -2.148522, -2.238381, + -1.765279, -1.833698, -1.684174, -1.374214, -1.259788, -1.473578, + -1.366698, -1.105641, -1.151468, -1.052101, + -0.3975942, -0.2708012, -0.1832284, -0.09187625, 0.001559797, 0.07204625, + 0.0783459, 0.09246785, 0.1097215, 0.1203732, + -0.5745555, -0.5160208, -0.4475087, -0.3612338, -0.3011235, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.151411, -2.133208, -2.145542, -2.198483, -2.335929, -2.401734, + -2.460874, -2.553266, -2.576532, -2.571083, + -2.586592, -2.559288, -2.461631, -2.440526, -2.46086, -2.4672, -2.609189, + -2.751828, -2.816938, -2.793565, + -2.410764, -2.404498, -2.467111, -2.314096, -1.887833, -2.113094, + -2.547507, -2.529567, -2.430087, -2.455852, + -2.803633, -2.214195, -2.758755, -3.378312, -3.186033, -3.65111, -4.398364, + -4.396877, -4.207345, -3.572534, + -5.248218, -4.912538, -4.553966, -4.296717, -4.063207, -4.280607, + -4.150732, -3.721014, -3.682456, -3.493971, + -0.4638011, -0.2794998, -0.1847423, -0.1013127, -0.01874243, 0.05095983, + 0.07407396, 0.09246785, 0.1097215, 0.1203732, + -0.5747783, -0.5167099, -0.4489113, -0.3624935, -0.3011871, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.046627, -2.075339, -2.133872, -2.192768, -2.29642, -2.385374, -2.421421, + -2.375845, -2.522865, -2.570514, + -2.329157, -2.661558, -2.550557, -2.222389, -2.452235, -2.64684, -2.825807, + -3.021084, -2.930932, -2.340356, + -4.449573, -5.811647, -6.238008, -5.904782, -5.536724, -5.204103, + -5.139505, -6.584569, -7.315174, -6.587632, + -7.759037, -7.923704, -7.657747, -7.342247, -7.189469, -6.957999, + -6.869977, -6.911683, -6.890206, -6.571945, + -5.766199, -5.629292, -5.500316, -5.457587, -5.137841, -4.946566, + -4.747363, -4.453594, -4.333026, -4.211693, + 1.063227, 1.156564, 1.210127, 1.265683, 1.325263, 1.37135, 1.379399, + 1.395321, 1.408047, 1.413615, + 0.8057297, 0.823011, 0.8517545, 0.8839636, 0.8734952, 0.8716547, 0.8792083, + 0.8959213, 0.933274, 0.9308779, + 0.3776965, 0.3511574, 0.3064211, 0.2771505, 0.2723292, 0.26952, 0.2818037, + 0.2319987, 0.165376, 0.1122594, + -0.2686788, -0.3904975, -0.4879307, -0.5713966, -0.6487127, -0.7579689, + -0.879656, -1.019151, -1.160378, -1.290107, + -1.44353, -1.544772, -1.67499, -1.812897, -1.98716, -2.274205, -2.556359, + -2.771371, -2.97445, -3.201544, + -3.11445, -3.268162, -3.425929, -3.644483, -3.970347, -4.194717, -4.426955, + -4.785159, -4.986393, -5.121875, + -4.842557, -4.942987, -5.010421, -5.153152, -5.254729, -5.326348, + -5.648958, -6.018217, -6.451276, -6.772595, + -6.110239, -6.028393, -5.95482, -5.914704, -5.883089, -5.694447, -5.721324, + -5.85588, -6.038979, -6.222065, + -6.000411, -6.046391, -5.883338, -5.710005, -5.577016, -5.360312, + -5.450391, -5.419651, -5.09429, -4.927251, + -4.122381, -4.0073, -3.920688, -3.890532, -3.619308, -3.348911, -3.164834, + -3.032901, -2.999816, -2.981941, + 1.021295, 1.058732, 1.055263, 1.055828, 1.06735, 1.085502, 1.072789, + 1.066372, 1.055719, 1.03737, + 0.5644462, 0.5258871, 0.505933, 0.5047766, 0.4665931, 0.4326979, 0.4054725, + 0.3554713, 0.3202047, 0.2826239, + -0.09909091, -0.1695052, -0.2423104, -0.308107, -0.3618009, -0.4362217, + -0.5038798, -0.5690737, -0.6322247, -0.6912587, + -0.8452172, -0.9907208, -1.12432, -1.275977, -1.407145, -1.514558, + -1.611202, -1.70489, -1.787341, -1.840466, + -1.660944, -1.836267, -2.043581, -2.208486, -2.365314, -2.544751, + -2.701359, -2.791034, -2.877407, -2.950873, + -2.49663, -2.691726, -2.876794, -3.089699, -3.315934, -3.422023, -3.487426, + -3.497974, -3.590583, -3.699841, + -3.409052, -3.454748, -3.579855, -3.723231, -3.777681, -3.849489, + -3.761075, -3.758978, -3.830392, -3.977781, + -4.332948, -4.30478, -4.268668, -4.142052, -4.038466, -3.906201, -3.853595, + -3.97838, -4.061214, -3.96847, + -4.037989, -3.923016, -3.725387, -3.504999, -3.335768, -3.048127, + -2.881483, -2.777315, -2.57696, -2.525698, + -2.675588, -2.521989, -2.357147, -2.217182, -1.92482, -1.59737, -1.311503, + -1.052147, -0.8554193, -0.7043628, + 0.868393, 0.8096983, 0.7342755, 0.6706711, 0.6231919, 0.5861326, 0.5235301, + 0.4832453, 0.4458269, 0.403123, + 0.3249699, 0.203376, 0.09710471, 0.005840228, -0.09959703, -0.186581, + -0.2659973, -0.3638777, -0.4377598, -0.5044919, + -0.4682975, -0.6448929, -0.7865676, -0.9014125, -1.010695, -1.141248, + -1.26495, -1.354475, -1.432743, -1.508723, + -1.302762, -1.487646, -1.667504, -1.870815, -2.038251, -2.165048, + -2.268736, -2.35599, -2.422225, -2.462221, + -1.897165, -2.089815, -2.293899, -2.467101, -2.626223, -2.795213, + -2.938092, -3.009527, -3.077491, -3.088423, + -2.338803, -2.44815, -2.576407, -2.714023, -2.881914, -2.990129, -3.088588, + -3.162073, -3.200975, -3.280721, + -2.629382, -2.617197, -2.644625, -2.714817, -2.728057, -2.764721, + -2.755495, -2.804715, -2.900975, -3.060241, + -2.806314, -2.699975, -2.604603, -2.467385, -2.316504, -2.181494, + -2.104434, -2.068235, -2.089983, -2.092961, + -2.350076, -2.191863, -2.015235, -1.808842, -1.591449, -1.298141, + -1.055869, -0.8611979, -0.6329716, -0.4672858, + -1.417739, -1.253469, -1.085745, -0.9032833, -0.6274295, -0.3408952, + -0.04521877, 0.2827266, 0.5437686, 0.7892594, + 0.08603396, -0.01244956, -0.1302382, -0.230692, -0.3224648, -0.4244072, + -0.5279509, -0.5862728, -0.6336403, -0.6853408, + -0.5083337, -0.6752201, -0.8401579, -1.021567, -1.198536, -1.330774, + -1.454516, -1.586487, -1.691136, -1.77504, + -1.209975, -1.465892, -1.672402, -1.836556, -2.003343, -2.175733, + -2.340954, -2.462788, -2.570125, -2.671328, + -1.835453, -2.040931, -2.250755, -2.488194, -2.693811, -2.848905, + -2.973454, -3.076011, -3.155657, -3.221975, + -2.087997, -2.262499, -2.42809, -2.583258, -2.728339, -2.874663, -3.012026, + -3.105144, -3.219588, -3.292633, + -2.219754, -2.239856, -2.291842, -2.338402, -2.416109, -2.477032, + -2.551712, -2.625169, -2.696625, -2.812562, + -2.214184, -2.120897, -2.02604, -1.978979, -1.914884, -1.861667, -1.797479, + -1.808812, -1.854385, -1.959785, + -2.145867, -1.937978, -1.761547, -1.585332, -1.406883, -1.215052, + -1.064301, -0.9400889, -0.883372, -0.8476374, + -1.6123, -1.449645, -1.277987, -1.08922, -0.8639328, -0.5785105, + -0.3019016, -0.06757384, 0.1769251, 0.3469639, + -0.4829444, -0.3573219, -0.2495783, -0.1331687, 0.03723535, 0.2394148, + 0.4937723, 0.8273596, 1.081453, 1.332125, + -0.8436456, -0.9486177, -1.059377, -1.153291, -1.241334, -1.344937, + -1.442711, -1.494026, -1.531425, -1.569082, + -1.363601, -1.517111, -1.673566, -1.853703, -2.024385, -2.153884, + -2.277348, -2.399244, -2.49864, -2.581057, + -1.900008, -2.121459, -2.298829, -2.446108, -2.600936, -2.75574, -2.91209, + -3.041326, -3.159974, -3.291958, + -2.332246, -2.485257, -2.649413, -2.831298, -2.996419, -3.138222, + -3.260611, -3.390049, -3.512205, -3.606771, + -2.484051, -2.582164, -2.656566, -2.747351, -2.843552, -2.953647, + -3.080211, -3.176885, -3.30392, -3.404524, + -2.524805, -2.482363, -2.478343, -2.466771, -2.482541, -2.493074, + -2.524661, -2.568022, -2.622594, -2.706606, + -2.361249, -2.253306, -2.134065, -2.054415, -1.964403, -1.889408, + -1.791761, -1.750293, -1.718029, -1.732389, + -2.014955, -1.839144, -1.68769, -1.535044, -1.384408, -1.211162, -1.050033, + -0.8907735, -0.7928692, -0.7119153, + -1.236629, -1.093662, -0.9508283, -0.8085872, -0.6442373, -0.4315634, + -0.1945382, -0.002699569, 0.1975364, 0.3391424, + -0.113028, 0.03701114, 0.1509672, 0.244096, 0.3440107, 0.4850455, 0.659207, + 0.8812482, 1.058411, 1.222257, + -1.854996, -1.890166, -1.915619, -1.932104, -1.946811, -1.971547, + -1.988159, -1.97958, -1.964028, -1.948233, + -2.224256, -2.271724, -2.324106, -2.393641, -2.449779, -2.481474, + -2.509733, -2.527462, -2.532758, -2.527077, + -2.520393, -2.60658, -2.663467, -2.701849, -2.744738, -2.780796, -2.817967, + -2.841355, -2.859165, -2.87995, + -2.696981, -2.736231, -2.781672, -2.833747, -2.878667, -2.9107, -2.931204, + -2.952544, -2.963497, -2.964039, + -2.669573, -2.676523, -2.663651, -2.663748, -2.672803, -2.703568, -2.73891, + -2.744817, -2.765156, -2.767472, + -2.476536, -2.429364, -2.405869, -2.377028, -2.36033, -2.332005, -2.306961, + -2.277795, -2.2701, -2.287573, + -2.063781, -1.986485, -1.90999, -1.85632, -1.792651, -1.720523, -1.636689, + -1.607912, -1.594598, -1.61015, + -1.519658, -1.429246, -1.343379, -1.238522, -1.116604, -1.015946, + -0.9358807, -0.8798366, -0.8500427, -0.8273263, + -0.7674065, -0.6452081, -0.5515807, -0.4693587, -0.3846785, -0.3116637, + -0.2242767, -0.1593899, -0.1116388, -0.06896764, + -0.01183324, 0.1091698, 0.2188423, 0.312474, 0.3854575, 0.4438081, + 0.5140151, 0.5824342, 0.6187019, 0.6379458, + -2.684902, -2.610719, -2.535948, -2.460751, -2.387049, -2.321111, + -2.254766, -2.182338, -2.107151, -2.031494, + -2.820069, -2.740268, -2.659949, -2.584899, -2.516798, -2.444206, + -2.371544, -2.295798, -2.221301, -2.142587, + -2.920424, -2.834318, -2.745224, -2.660668, -2.577662, -2.494831, -2.4213, + -2.345643, -2.268257, -2.192751, + -2.919983, -2.838565, -2.757891, -2.677184, -2.599998, -2.519447, + -2.432564, -2.343707, -2.255901, -2.170567, + -2.795936, -2.718457, -2.634572, -2.554108, -2.473378, -2.391261, + -2.312509, -2.233216, -2.158018, -2.077407, + -2.562569, -2.481418, -2.403707, -2.320077, -2.236581, -2.159199, + -2.083839, -2.011575, -1.949767, -1.892647, + -2.250703, -2.162687, -2.078802, -2.00374, -1.924842, -1.843397, -1.762613, + -1.697863, -1.633513, -1.583002, + -1.825546, -1.773501, -1.720045, -1.646263, -1.569409, -1.491826, + -1.415384, -1.344221, -1.289576, -1.235646, + -1.16891, -1.113447, -1.082653, -1.06613, -1.038724, -1.014261, -0.971841, + -0.9325086, -0.8841423, -0.8362257, + -0.4466377, -0.3950162, -0.3640113, -0.3467484, -0.3497896, -0.3435763, + -0.3285452, -0.3093777, -0.3015329, -0.2992152, + -3.47265, -3.349887, -3.225127, -3.099491, -2.975098, -2.858005, -2.742823, + -2.620348, -2.497191, -2.375009, + -3.669024, -3.520918, -3.372771, -3.233594, -3.098513, -2.952549, + -2.805939, -2.665119, -2.521485, -2.376583, + -3.875113, -3.713779, -3.561024, -3.408706, -3.2558, -3.100774, -2.95319, + -2.807744, -2.659667, -2.510706, + -3.950386, -3.795719, -3.641412, -3.475792, -3.323338, -3.181724, + -3.039761, -2.904638, -2.775108, -2.650002, + -3.947389, -3.783504, -3.606038, -3.452417, -3.30665, -3.171729, -3.049297, + -2.93359, -2.822966, -2.726708, + -3.906445, -3.746139, -3.594896, -3.442577, -3.297046, -3.1648, -3.036906, + -2.916452, -2.812173, -2.712864, + -3.83857, -3.71836, -3.602963, -3.481781, -3.346611, -3.205612, -3.061509, + -2.923938, -2.794568, -2.691266, + -3.559861, -3.530004, -3.478284, -3.396861, -3.302307, -3.188716, + -3.069149, -2.94336, -2.817003, -2.663864, + -2.915129, -2.919575, -2.929287, -2.93514, -2.908199, -2.866826, -2.797559, + -2.709057, -2.578434, -2.428774, + -2.010599, -2.042224, -2.077676, -2.104624, -2.122853, -2.120454, + -2.097497, -2.047853, -1.992431, -1.92157, + -4.22112, -4.07559, -3.922843, -3.768222, -3.614719, -3.473573, -3.340809, + -3.200793, -3.061464, -2.926893, + -4.502892, -4.344226, -4.183433, -4.026276, -3.878217, -3.729609, -3.58069, + -3.431968, -3.287462, -3.145941, + -4.631217, -4.473382, -4.315218, -4.164486, -4.013389, -3.85793, -3.710944, + -3.570176, -3.429156, -3.28708, + -4.626989, -4.476557, -4.326241, -4.174338, -4.032887, -3.895635, + -3.756263, -3.619544, -3.485802, -3.356973, + -4.556738, -4.395508, -4.235921, -4.096162, -3.958964, -3.832275, + -3.715655, -3.600988, -3.489401, -3.387974, + -4.468916, -4.302645, -4.15163, -4.001896, -3.865624, -3.743882, -3.626797, + -3.521506, -3.429111, -3.338457, + -4.44977, -4.292527, -4.144361, -4.001679, -3.853956, -3.707088, -3.567008, + -3.438192, -3.317925, -3.220261, + -4.439157, -4.329401, -4.202333, -4.056956, -3.905393, -3.736642, + -3.568599, -3.403389, -3.260043, -3.114572, + -4.203069, -4.12499, -4.031964, -3.926179, -3.793952, -3.63649, -3.470912, + -3.312045, -3.139288, -2.968029, + -3.549062, -3.518938, -3.466148, -3.407319, -3.308811, -3.204609, + -3.090285, -2.963688, -2.832902, -2.702063, + -4.634931, -4.510322, -4.387384, -4.262967, -4.13814, -4.019274, -3.907151, + -3.793168, -3.677498, -3.562162, + -4.812496, -4.685526, -4.557202, -4.432652, -4.315062, -4.196854, + -4.077685, -3.961596, -3.849445, -3.737238, + -4.926235, -4.801914, -4.682349, -4.564368, -4.445381, -4.328474, + -4.217326, -4.105275, -3.991226, -3.87654, + -4.974308, -4.86126, -4.746056, -4.629657, -4.521437, -4.412365, -4.300677, + -4.186651, -4.075935, -3.968378, + -4.938999, -4.831478, -4.722439, -4.619403, -4.516479, -4.414182, + -4.313823, -4.215149, -4.114635, -4.01406, + -4.802555, -4.709178, -4.621257, -4.533046, -4.446228, -4.362994, + -4.278905, -4.198927, -4.114691, -4.029364, + -4.646967, -4.549815, -4.467683, -4.393793, -4.318109, -4.249639, + -4.184724, -4.118573, -4.056467, -3.996501, + -4.553621, -4.461698, -4.370646, -4.279413, -4.197869, -4.108099, + -4.025449, -3.951957, -3.882595, -3.806807, + -4.450593, -4.350024, -4.254059, -4.160119, -4.047391, -3.930037, + -3.814608, -3.705483, -3.598191, -3.498837, + -4.176404, -4.088185, -3.99167, -3.888712, -3.763858, -3.636919, -3.503574, + -3.371032, -3.253147, -3.15005, + -5.305916, -5.228198, -5.149081, -5.066242, -4.979733, -4.891483, + -4.805131, -4.715399, -4.625538, -4.536609, + -5.409827, -5.338651, -5.264568, -5.190886, -5.118539, -5.040178, + -4.958767, -4.879787, -4.798156, -4.714354, + -5.436405, -5.377537, -5.321352, -5.260819, -5.19785, -5.135309, -5.069129, + -4.99902, -4.925947, -4.848398, + -5.425586, -5.371675, -5.315626, -5.258965, -5.205391, -5.147702, + -5.090356, -5.029489, -4.966897, -4.903516, + -5.421552, -5.364684, -5.306411, -5.247279, -5.189979, -5.13738, -5.082496, + -5.023454, -4.961823, -4.898187, + -5.399646, -5.353481, -5.305019, -5.255711, -5.203397, -5.146911, + -5.086962, -5.023842, -4.958927, -4.891607, + -5.324206, -5.285987, -5.253324, -5.216431, -5.17214, -5.119621, -5.066675, + -5.011673, -4.952978, -4.889044, + -5.201594, -5.185238, -5.161349, -5.129875, -5.091774, -5.047886, + -5.002157, -4.954802, -4.903096, -4.845263, + -5.027672, -5.009829, -4.996872, -4.982873, -4.957479, -4.928816, + -4.894786, -4.855186, -4.809749, -4.755376, + -4.800905, -4.790343, -4.778637, -4.76788, -4.754467, -4.730603, -4.70276, + -4.672376, -4.637187, -4.600635, + -5.877733, -5.806942, -5.737911, -5.665781, -5.589697, -5.508978, + -5.428902, -5.346344, -5.262498, -5.177478, + -5.999748, -5.93775, -5.871769, -5.80268, -5.736236, -5.664495, -5.589656, + -5.515277, -5.436458, -5.355067, + -6.105768, -6.050148, -5.99489, -5.934047, -5.869964, -5.807178, -5.740132, + -5.669778, -5.59723, -5.518231, + -6.204219, -6.156556, -6.104576, -6.049919, -5.996562, -5.93629, -5.873829, + -5.805651, -5.734739, -5.662208, + -6.283307, -6.238613, -6.188774, -6.136417, -6.083857, -6.031115, + -5.974864, -5.914575, -5.849953, -5.782243, + -6.327734, -6.29085, -6.250577, -6.207967, -6.160203, -6.109562, -6.055408, + -5.996362, -5.935081, -5.871264, + -6.328987, -6.301887, -6.276907, -6.244695, -6.205217, -6.156336, + -6.106204, -6.054013, -5.999475, -5.941885, + -6.286625, -6.282731, -6.267518, -6.242774, -6.20835, -6.168219, -6.125497, + -6.080946, -6.031807, -5.975547, + -6.193247, -6.187342, -6.183197, -6.175198, -6.155711, -6.131026, -6.09807, + -6.059319, -6.013174, -5.959172, + -6.058994, -6.055746, -6.05086, -6.046446, -6.038727, -6.018096, -5.98981, + -5.954857, -5.918507, -5.881165, + -6.561323, -6.499266, -6.438377, -6.375708, -6.310773, -6.244139, + -6.180099, -6.115896, -6.051152, -5.986147, + -6.845045, -6.786884, -6.725439, -6.661109, -6.601454, -6.540991, + -6.479097, -6.417562, -6.355603, -6.293946, + -7.102453, -7.045627, -6.989235, -6.932842, -6.874574, -6.816972, + -6.758873, -6.701811, -6.644189, -6.583754, + -7.351357, -7.301179, -7.248499, -7.19398, -7.142448, -7.089523, -7.035904, + -6.981055, -6.926404, -6.871471, + -7.598204, -7.549111, -7.497645, -7.447221, -7.397934, -7.350796, + -7.304009, -7.256029, -7.20568, -7.155462, + -7.82113, -7.78062, -7.739799, -7.69751, -7.652907, -7.609156, -7.56383, + -7.517503, -7.472572, -7.428266, + -8.020307, -7.985536, -7.955499, -7.923059, -7.885509, -7.841635, + -7.801394, -7.76416, -7.727195, -7.691062, + -8.203031, -8.185472, -8.161825, -8.130711, -8.093474, -8.056222, + -8.020072, -7.988377, -7.958443, -7.925798, + -8.343966, -8.324842, -8.308795, -8.291328, -8.267859, -8.240988, -8.21365, + -8.186219, -8.156057, -8.126547, + -8.44836, -8.431485, -8.415438, -8.404657, -8.391961, -8.373449, -8.350844, + -8.326425, -8.308137, -8.293455, + -7.418903, -7.380289, -7.343199, -7.30528, -7.266068, -7.225653, -7.186882, + -7.148771, -7.1104, -7.071782, + -7.886793, -7.851552, -7.814201, -7.774897, -7.739104, -7.703277, + -7.666594, -7.630272, -7.594023, -7.558314, + -8.341415, -8.308264, -8.274553, -8.240778, -8.205718, -8.171395, + -8.137189, -8.103981, -8.07057, -8.035443, + -8.796901, -8.767877, -8.73701, -8.704835, -8.674766, -8.644069, -8.61279, + -8.580782, -8.54905, -8.517601, + -9.253543, -9.225341, -9.195226, -9.165483, -9.137247, -9.111868, + -9.086359, -9.059315, -9.030994, -9.003092, + -9.699064, -9.677286, -9.654745, -9.63162, -9.607246, -9.582701, -9.557117, + -9.531214, -9.505648, -9.481256, + -10.12736, -10.10717, -10.09253, -10.07685, -10.05586, -10.03019, + -10.00788, -9.988334, -9.96856, -9.949014, + -10.53594, -10.52901, -10.51833, -10.50091, -10.47864, -10.45729, + -10.43758, -10.42123, -10.40631, -10.38915, + -10.90329, -10.89263, -10.88709, -10.88161, -10.87157, -10.86071, + -10.84752, -10.83391, -10.81815, -10.80181, + -11.248, -11.23781, -11.23062, -11.22926, -11.23034, -11.22393, -11.21365, + -11.20186, -11.19413, -11.18977, + -13.83239, -13.82074, -13.808, -13.79507, -13.78222, -13.76985, -13.75674, + -13.74387, -13.73124, -13.71895, + -14.99369, -14.98229, -14.97097, -14.96008, -14.94822, -14.9366, -14.92516, + -14.9145, -14.90532, -14.89437, + -16.12242, -16.11533, -16.10389, -16.09248, -16.08173, -16.06978, + -16.06084, -16.05067, -16.03975, -16.03102, + -17.2117, -17.20174, -17.1923, -17.18274, -17.17234, -17.16337, -17.15279, + -17.14514, -17.13551, -17.12395, + -18.23167, -18.22563, -18.21396, -18.20369, -18.19536, -18.19025, -18.1865, + -18.17599, -18.1694, -18.16037, + -19.16084, -19.15458, -19.15068, -19.14543, -19.14352, -19.13608, + -19.12984, -19.12499, -19.11406, -19.10741, + -19.96635, -19.96284, -19.96158, -19.96541, -19.9607, -19.95777, -19.95134, + -19.94964, -19.94552, -19.9427, + -20.64782, -20.64862, -20.65172, -20.64795, -20.64328, -20.63709, -20.6386, + -20.63893, -20.64199, -20.63851, + -21.1739, -21.17439, -21.17778, -21.18302, -21.18665, -21.19431, -21.1958, + -21.1981, -21.19873, -21.19386, + -21.59861, -21.59514, -21.59659, -21.60578, -21.61945, -21.62407, + -21.62483, -21.62446, -21.6267, -21.63452 ; + + bnds = 1, 2 ; + + lat_bnds = + -80, -79, + -79, -78, + -78, -77, + -77, -76, + -76, -75, + -75, -74, + -74, -73, + -73, -72, + -72, -71, + -71, -70 ; + + lon_bnds = + 12.5, 13.75, + 13.75, 15, + 15, 16.25, + 16.25, 17.5, + 17.5, 18.75, + 18.75, 20, + 20, 21.25, + 21.25, 22.5, + 22.5, 23.75, + 23.75, 25 ; + + time_bnds = + 9497, 9528 ; +} diff --git a/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_masked.cdl b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_masked.cdl new file mode 100644 index 0000000..d3276fc --- /dev/null +++ b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_masked.cdl @@ -0,0 +1,453 @@ +netcdf atmos_cmip.ua_masked { +dimensions: + time = UNLIMITED ; // (1 currently) + plev19 = 19 ; + lat = 10 ; + lon = 10 ; + bnds = 2 ; +variables: + double plev19(plev19) ; + plev19:_FillValue = NaN ; + double time(time) ; + time:_FillValue = NaN ; + time:long_name = "time" ; + time:axis = "T" ; + time:calendar_type = "JULIAN" ; + time:bounds = "time_bnds" ; + time:units = "days since 1979-01-01" ; + time:calendar = "julian" ; + double lat(lat) ; + lat:_FillValue = NaN ; + lat:long_name = "latitude" ; + lat:units = "degrees_N" ; + lat:axis = "Y" ; + lat:bounds = "lat_bnds" ; + double lon(lon) ; + lon:_FillValue = NaN ; + lon:long_name = "longitude" ; + lon:units = "degrees_E" ; + lon:axis = "X" ; + lon:bounds = "lon_bnds" ; + float ua_unmsk(time, plev19, lat, lon) ; + ua_unmsk:_FillValue = 1.e+20f ; + ua_unmsk:units = "m s-1" ; + ua_unmsk:long_name = "Eastward Wind" ; + ua_unmsk:cell_methods = "time: mean" ; + ua_unmsk:cell_measures = "area: area" ; + ua_unmsk:standard_name = "eastward_wind" ; + ua_unmsk:interp_method = "conserve_order2" ; + ua_unmsk:pressure_mask = "True" ; + ua_unmsk:missing_value = 1.e+20 ; + double bnds(bnds) ; + bnds:_FillValue = NaN ; + bnds:long_name = "vertex number" ; + double lat_bnds(lat, bnds) ; + lat_bnds:_FillValue = NaN ; + lat_bnds:long_name = "latitude bounds" ; + double lon_bnds(lon, bnds) ; + lon_bnds:_FillValue = NaN ; + lon_bnds:long_name = "longitude bounds" ; + double time_bnds(time, bnds) ; + time_bnds:_FillValue = NaN ; + time_bnds:long_name = "time axis boundaries" ; + +// global attributes: + :title = "c96L65_am5f9d7r0_amip" ; + :associated_files = "area: 20050101.grid_spec.nc" ; + :grid_type = "regular" ; + :grid_tile = "N/A" ; + :code_release_version = "2024.05" ; + :git_hash = "5d306c05d9fe755cab04eedc8fd3de0d3c8355a0" ; + :creationtime = "Thu Jun 26 22:27:29 2025" ; + :hostname = "pp208" ; + :history = "Tue Jul 1 23:47:57 2025: ncks -d lat,10,19 -d lon,10,19 -d time,0,0 atmos_cmip.200501-200512.ua_unmsk.nc atmos_cmip.ua_unmsk.nc\nfregrid --standard_dimension --input_mosaic C96_mosaic.nc --input_file 20050101.atmos_month_cmip --interp_method conserve_order2 --remap_file .fregrid_remap_file_288_by_180.nc --nlon 288 --nlat 180 --scalar_field tas,ts,psl,ps,uas,height10m,vas,sfcWind,hurs,height2m,huss,pr,prsn,prc,evspsbl,tauu,tauv,hfls,hfss,rlds,rlus,rsds,rsus,rsdscs,rsuscs,rldscs,rsdt,rsut,rlut,rlutcs,rsutcs,prw,clt,clwvi,clivi,rtmt,ccb,cct,ci,sci,ta_unmsk,ua_unmsk,va_unmsk,hus_unmsk,hur_unmsk,wap_unmsk,zg_unmsk,ap,b,ap_bnds,b_bnds,lev_bnds,utendnogw,utendogw,time_bnds --output_file out.nc" ; + :external_variables = "area" ; + :NCO = "netCDF Operators version 5.2.4 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco, Citation = 10.1016/j.envsoft.2008.03.004)" ; +data: + + plev19 = 100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, + 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000, 500, 100 ; + + time = 9512.5 ; + + lat = -79.5, -78.5, -77.5, -76.5, -75.5, -74.5, -73.5, -72.5, -71.5, -70.5 ; + + lon = 13.125, 14.375, 15.625, 16.875, 18.125, 19.375, 20.625, 21.875, + 23.125, 24.375 ; + + ua_unmsk = + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + -1.765279, -1.833698, -1.684174, -1.374214, -1.259788, -1.473578, + -1.366698, -1.105641, -1.151468, -1.052101, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, -3.65111, -4.398364, -4.396877, -4.207345, -3.572534, + -5.248218, -4.912538, -4.553966, -4.296717, -4.063207, -4.280607, + -4.150732, -3.721014, -3.682456, -3.493971, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + _, _, _, _, _, _, _, _, _, _, + -4.449573, -5.811647, -6.238008, -5.904782, -5.536724, -5.204103, + -5.139505, -6.584569, -7.315174, -6.587632, + -7.759037, -7.923704, -7.657747, -7.342247, -7.189469, -6.957999, + -6.869977, -6.911683, -6.890206, -6.571945, + -5.766199, -5.629292, -5.500316, -5.457587, -5.137841, -4.946566, + -4.747363, -4.453594, -4.333026, -4.211693, + 1.063227, 1.156564, 1.210127, 1.265683, 1.325263, 1.37135, 1.379399, + 1.395321, 1.408047, 1.413615, + 0.8057297, 0.823011, 0.8517545, 0.8839636, 0.8734952, 0.8716547, 0.8792083, + 0.8959213, 0.933274, 0.9308779, + 0.3776965, 0.3511574, 0.3064211, 0.2771505, 0.2723292, 0.26952, 0.2818037, + 0.2319987, 0.165376, 0.1122594, + -0.2686788, -0.3904975, -0.4879307, -0.5713966, -0.6487127, -0.7579689, + -0.879656, -1.019151, -1.160378, -1.290107, + -1.44353, -1.544772, -1.67499, -1.812897, -1.98716, -2.274205, -2.556359, + -2.771371, -2.97445, -3.201544, + -3.11445, -3.268162, -3.425929, -3.644483, -3.970347, -4.194717, -4.426955, + -4.785159, -4.986393, -5.121875, + -4.842557, -4.942987, -5.010421, -5.153152, -5.254729, -5.326348, + -5.648958, -6.018217, -6.451276, -6.772595, + -6.110239, -6.028393, -5.95482, -5.914704, -5.883089, -5.694447, -5.721324, + -5.85588, -6.038979, -6.222065, + -6.000411, -6.046391, -5.883338, -5.710005, -5.577016, -5.360312, + -5.450391, -5.419651, -5.09429, -4.927251, + -4.122381, -4.0073, -3.920688, -3.890532, -3.619308, -3.348911, -3.164834, + -3.032901, -2.999816, -2.981941, + 1.021295, 1.058732, 1.055263, 1.055828, 1.06735, 1.085502, 1.072789, + 1.066372, 1.055719, 1.03737, + 0.5644462, 0.5258871, 0.505933, 0.5047766, 0.4665931, 0.4326979, 0.4054725, + 0.3554713, 0.3202047, 0.2826239, + -0.09909091, -0.1695052, -0.2423104, -0.308107, -0.3618009, -0.4362217, + -0.5038798, -0.5690737, -0.6322247, -0.6912587, + -0.8452172, -0.9907208, -1.12432, -1.275977, -1.407145, -1.514558, + -1.611202, -1.70489, -1.787341, -1.840466, + -1.660944, -1.836267, -2.043581, -2.208486, -2.365314, -2.544751, + -2.701359, -2.791034, -2.877407, -2.950873, + -2.49663, -2.691726, -2.876794, -3.089699, -3.315934, -3.422023, -3.487426, + -3.497974, -3.590583, -3.699841, + -3.409052, -3.454748, -3.579855, -3.723231, -3.777681, -3.849489, + -3.761075, -3.758978, -3.830392, -3.977781, + -4.332948, -4.30478, -4.268668, -4.142052, -4.038466, -3.906201, -3.853595, + -3.97838, -4.061214, -3.96847, + -4.037989, -3.923016, -3.725387, -3.504999, -3.335768, -3.048127, + -2.881483, -2.777315, -2.57696, -2.525698, + -2.675588, -2.521989, -2.357147, -2.217182, -1.92482, -1.59737, -1.311503, + -1.052147, -0.8554193, -0.7043628, + 0.868393, 0.8096983, 0.7342755, 0.6706711, 0.6231919, 0.5861326, 0.5235301, + 0.4832453, 0.4458269, 0.403123, + 0.3249699, 0.203376, 0.09710471, 0.005840228, -0.09959703, -0.186581, + -0.2659973, -0.3638777, -0.4377598, -0.5044919, + -0.4682975, -0.6448929, -0.7865676, -0.9014125, -1.010695, -1.141248, + -1.26495, -1.354475, -1.432743, -1.508723, + -1.302762, -1.487646, -1.667504, -1.870815, -2.038251, -2.165048, + -2.268736, -2.35599, -2.422225, -2.462221, + -1.897165, -2.089815, -2.293899, -2.467101, -2.626223, -2.795213, + -2.938092, -3.009527, -3.077491, -3.088423, + -2.338803, -2.44815, -2.576407, -2.714023, -2.881914, -2.990129, -3.088588, + -3.162073, -3.200975, -3.280721, + -2.629382, -2.617197, -2.644625, -2.714817, -2.728057, -2.764721, + -2.755495, -2.804715, -2.900975, -3.060241, + -2.806314, -2.699975, -2.604603, -2.467385, -2.316504, -2.181494, + -2.104434, -2.068235, -2.089983, -2.092961, + -2.350076, -2.191863, -2.015235, -1.808842, -1.591449, -1.298141, + -1.055869, -0.8611979, -0.6329716, -0.4672858, + -1.417739, -1.253469, -1.085745, -0.9032833, -0.6274295, -0.3408952, + -0.04521877, 0.2827266, 0.5437686, 0.7892594, + 0.08603396, -0.01244956, -0.1302382, -0.230692, -0.3224648, -0.4244072, + -0.5279509, -0.5862728, -0.6336403, -0.6853408, + -0.5083337, -0.6752201, -0.8401579, -1.021567, -1.198536, -1.330774, + -1.454516, -1.586487, -1.691136, -1.77504, + -1.209975, -1.465892, -1.672402, -1.836556, -2.003343, -2.175733, + -2.340954, -2.462788, -2.570125, -2.671328, + -1.835453, -2.040931, -2.250755, -2.488194, -2.693811, -2.848905, + -2.973454, -3.076011, -3.155657, -3.221975, + -2.087997, -2.262499, -2.42809, -2.583258, -2.728339, -2.874663, -3.012026, + -3.105144, -3.219588, -3.292633, + -2.219754, -2.239856, -2.291842, -2.338402, -2.416109, -2.477032, + -2.551712, -2.625169, -2.696625, -2.812562, + -2.214184, -2.120897, -2.02604, -1.978979, -1.914884, -1.861667, -1.797479, + -1.808812, -1.854385, -1.959785, + -2.145867, -1.937978, -1.761547, -1.585332, -1.406883, -1.215052, + -1.064301, -0.9400889, -0.883372, -0.8476374, + -1.6123, -1.449645, -1.277987, -1.08922, -0.8639328, -0.5785105, + -0.3019016, -0.06757384, 0.1769251, 0.3469639, + -0.4829444, -0.3573219, -0.2495783, -0.1331687, 0.03723535, 0.2394148, + 0.4937723, 0.8273596, 1.081453, 1.332125, + -0.8436456, -0.9486177, -1.059377, -1.153291, -1.241334, -1.344937, + -1.442711, -1.494026, -1.531425, -1.569082, + -1.363601, -1.517111, -1.673566, -1.853703, -2.024385, -2.153884, + -2.277348, -2.399244, -2.49864, -2.581057, + -1.900008, -2.121459, -2.298829, -2.446108, -2.600936, -2.75574, -2.91209, + -3.041326, -3.159974, -3.291958, + -2.332246, -2.485257, -2.649413, -2.831298, -2.996419, -3.138222, + -3.260611, -3.390049, -3.512205, -3.606771, + -2.484051, -2.582164, -2.656566, -2.747351, -2.843552, -2.953647, + -3.080211, -3.176885, -3.30392, -3.404524, + -2.524805, -2.482363, -2.478343, -2.466771, -2.482541, -2.493074, + -2.524661, -2.568022, -2.622594, -2.706606, + -2.361249, -2.253306, -2.134065, -2.054415, -1.964403, -1.889408, + -1.791761, -1.750293, -1.718029, -1.732389, + -2.014955, -1.839144, -1.68769, -1.535044, -1.384408, -1.211162, -1.050033, + -0.8907735, -0.7928692, -0.7119153, + -1.236629, -1.093662, -0.9508283, -0.8085872, -0.6442373, -0.4315634, + -0.1945382, -0.002699569, 0.1975364, 0.3391424, + -0.113028, 0.03701114, 0.1509672, 0.244096, 0.3440107, 0.4850455, 0.659207, + 0.8812482, 1.058411, 1.222257, + -1.854996, -1.890166, -1.915619, -1.932104, -1.946811, -1.971547, + -1.988159, -1.97958, -1.964028, -1.948233, + -2.224256, -2.271724, -2.324106, -2.393641, -2.449779, -2.481474, + -2.509733, -2.527462, -2.532758, -2.527077, + -2.520393, -2.60658, -2.663467, -2.701849, -2.744738, -2.780796, -2.817967, + -2.841355, -2.859165, -2.87995, + -2.696981, -2.736231, -2.781672, -2.833747, -2.878667, -2.9107, -2.931204, + -2.952544, -2.963497, -2.964039, + -2.669573, -2.676523, -2.663651, -2.663748, -2.672803, -2.703568, -2.73891, + -2.744817, -2.765156, -2.767472, + -2.476536, -2.429364, -2.405869, -2.377028, -2.36033, -2.332005, -2.306961, + -2.277795, -2.2701, -2.287573, + -2.063781, -1.986485, -1.90999, -1.85632, -1.792651, -1.720523, -1.636689, + -1.607912, -1.594598, -1.61015, + -1.519658, -1.429246, -1.343379, -1.238522, -1.116604, -1.015946, + -0.9358807, -0.8798366, -0.8500427, -0.8273263, + -0.7674065, -0.6452081, -0.5515807, -0.4693587, -0.3846785, -0.3116637, + -0.2242767, -0.1593899, -0.1116388, -0.06896764, + -0.01183324, 0.1091698, 0.2188423, 0.312474, 0.3854575, 0.4438081, + 0.5140151, 0.5824342, 0.6187019, 0.6379458, + -2.684902, -2.610719, -2.535948, -2.460751, -2.387049, -2.321111, + -2.254766, -2.182338, -2.107151, -2.031494, + -2.820069, -2.740268, -2.659949, -2.584899, -2.516798, -2.444206, + -2.371544, -2.295798, -2.221301, -2.142587, + -2.920424, -2.834318, -2.745224, -2.660668, -2.577662, -2.494831, -2.4213, + -2.345643, -2.268257, -2.192751, + -2.919983, -2.838565, -2.757891, -2.677184, -2.599998, -2.519447, + -2.432564, -2.343707, -2.255901, -2.170567, + -2.795936, -2.718457, -2.634572, -2.554108, -2.473378, -2.391261, + -2.312509, -2.233216, -2.158018, -2.077407, + -2.562569, -2.481418, -2.403707, -2.320077, -2.236581, -2.159199, + -2.083839, -2.011575, -1.949767, -1.892647, + -2.250703, -2.162687, -2.078802, -2.00374, -1.924842, -1.843397, -1.762613, + -1.697863, -1.633513, -1.583002, + -1.825546, -1.773501, -1.720045, -1.646263, -1.569409, -1.491826, + -1.415384, -1.344221, -1.289576, -1.235646, + -1.16891, -1.113447, -1.082653, -1.06613, -1.038724, -1.014261, -0.971841, + -0.9325086, -0.8841423, -0.8362257, + -0.4466377, -0.3950162, -0.3640113, -0.3467484, -0.3497896, -0.3435763, + -0.3285452, -0.3093777, -0.3015329, -0.2992152, + -3.47265, -3.349887, -3.225127, -3.099491, -2.975098, -2.858005, -2.742823, + -2.620348, -2.497191, -2.375009, + -3.669024, -3.520918, -3.372771, -3.233594, -3.098513, -2.952549, + -2.805939, -2.665119, -2.521485, -2.376583, + -3.875113, -3.713779, -3.561024, -3.408706, -3.2558, -3.100774, -2.95319, + -2.807744, -2.659667, -2.510706, + -3.950386, -3.795719, -3.641412, -3.475792, -3.323338, -3.181724, + -3.039761, -2.904638, -2.775108, -2.650002, + -3.947389, -3.783504, -3.606038, -3.452417, -3.30665, -3.171729, -3.049297, + -2.93359, -2.822966, -2.726708, + -3.906445, -3.746139, -3.594896, -3.442577, -3.297046, -3.1648, -3.036906, + -2.916452, -2.812173, -2.712864, + -3.83857, -3.71836, -3.602963, -3.481781, -3.346611, -3.205612, -3.061509, + -2.923938, -2.794568, -2.691266, + -3.559861, -3.530004, -3.478284, -3.396861, -3.302307, -3.188716, + -3.069149, -2.94336, -2.817003, -2.663864, + -2.915129, -2.919575, -2.929287, -2.93514, -2.908199, -2.866826, -2.797559, + -2.709057, -2.578434, -2.428774, + -2.010599, -2.042224, -2.077676, -2.104624, -2.122853, -2.120454, + -2.097497, -2.047853, -1.992431, -1.92157, + -4.22112, -4.07559, -3.922843, -3.768222, -3.614719, -3.473573, -3.340809, + -3.200793, -3.061464, -2.926893, + -4.502892, -4.344226, -4.183433, -4.026276, -3.878217, -3.729609, -3.58069, + -3.431968, -3.287462, -3.145941, + -4.631217, -4.473382, -4.315218, -4.164486, -4.013389, -3.85793, -3.710944, + -3.570176, -3.429156, -3.28708, + -4.626989, -4.476557, -4.326241, -4.174338, -4.032887, -3.895635, + -3.756263, -3.619544, -3.485802, -3.356973, + -4.556738, -4.395508, -4.235921, -4.096162, -3.958964, -3.832275, + -3.715655, -3.600988, -3.489401, -3.387974, + -4.468916, -4.302645, -4.15163, -4.001896, -3.865624, -3.743882, -3.626797, + -3.521506, -3.429111, -3.338457, + -4.44977, -4.292527, -4.144361, -4.001679, -3.853956, -3.707088, -3.567008, + -3.438192, -3.317925, -3.220261, + -4.439157, -4.329401, -4.202333, -4.056956, -3.905393, -3.736642, + -3.568599, -3.403389, -3.260043, -3.114572, + -4.203069, -4.12499, -4.031964, -3.926179, -3.793952, -3.63649, -3.470912, + -3.312045, -3.139288, -2.968029, + -3.549062, -3.518938, -3.466148, -3.407319, -3.308811, -3.204609, + -3.090285, -2.963688, -2.832902, -2.702063, + -4.634931, -4.510322, -4.387384, -4.262967, -4.13814, -4.019274, -3.907151, + -3.793168, -3.677498, -3.562162, + -4.812496, -4.685526, -4.557202, -4.432652, -4.315062, -4.196854, + -4.077685, -3.961596, -3.849445, -3.737238, + -4.926235, -4.801914, -4.682349, -4.564368, -4.445381, -4.328474, + -4.217326, -4.105275, -3.991226, -3.87654, + -4.974308, -4.86126, -4.746056, -4.629657, -4.521437, -4.412365, -4.300677, + -4.186651, -4.075935, -3.968378, + -4.938999, -4.831478, -4.722439, -4.619403, -4.516479, -4.414182, + -4.313823, -4.215149, -4.114635, -4.01406, + -4.802555, -4.709178, -4.621257, -4.533046, -4.446228, -4.362994, + -4.278905, -4.198927, -4.114691, -4.029364, + -4.646967, -4.549815, -4.467683, -4.393793, -4.318109, -4.249639, + -4.184724, -4.118573, -4.056467, -3.996501, + -4.553621, -4.461698, -4.370646, -4.279413, -4.197869, -4.108099, + -4.025449, -3.951957, -3.882595, -3.806807, + -4.450593, -4.350024, -4.254059, -4.160119, -4.047391, -3.930037, + -3.814608, -3.705483, -3.598191, -3.498837, + -4.176404, -4.088185, -3.99167, -3.888712, -3.763858, -3.636919, -3.503574, + -3.371032, -3.253147, -3.15005, + -5.305916, -5.228198, -5.149081, -5.066242, -4.979733, -4.891483, + -4.805131, -4.715399, -4.625538, -4.536609, + -5.409827, -5.338651, -5.264568, -5.190886, -5.118539, -5.040178, + -4.958767, -4.879787, -4.798156, -4.714354, + -5.436405, -5.377537, -5.321352, -5.260819, -5.19785, -5.135309, -5.069129, + -4.99902, -4.925947, -4.848398, + -5.425586, -5.371675, -5.315626, -5.258965, -5.205391, -5.147702, + -5.090356, -5.029489, -4.966897, -4.903516, + -5.421552, -5.364684, -5.306411, -5.247279, -5.189979, -5.13738, -5.082496, + -5.023454, -4.961823, -4.898187, + -5.399646, -5.353481, -5.305019, -5.255711, -5.203397, -5.146911, + -5.086962, -5.023842, -4.958927, -4.891607, + -5.324206, -5.285987, -5.253324, -5.216431, -5.17214, -5.119621, -5.066675, + -5.011673, -4.952978, -4.889044, + -5.201594, -5.185238, -5.161349, -5.129875, -5.091774, -5.047886, + -5.002157, -4.954802, -4.903096, -4.845263, + -5.027672, -5.009829, -4.996872, -4.982873, -4.957479, -4.928816, + -4.894786, -4.855186, -4.809749, -4.755376, + -4.800905, -4.790343, -4.778637, -4.76788, -4.754467, -4.730603, -4.70276, + -4.672376, -4.637187, -4.600635, + -5.877733, -5.806942, -5.737911, -5.665781, -5.589697, -5.508978, + -5.428902, -5.346344, -5.262498, -5.177478, + -5.999748, -5.93775, -5.871769, -5.80268, -5.736236, -5.664495, -5.589656, + -5.515277, -5.436458, -5.355067, + -6.105768, -6.050148, -5.99489, -5.934047, -5.869964, -5.807178, -5.740132, + -5.669778, -5.59723, -5.518231, + -6.204219, -6.156556, -6.104576, -6.049919, -5.996562, -5.93629, -5.873829, + -5.805651, -5.734739, -5.662208, + -6.283307, -6.238613, -6.188774, -6.136417, -6.083857, -6.031115, + -5.974864, -5.914575, -5.849953, -5.782243, + -6.327734, -6.29085, -6.250577, -6.207967, -6.160203, -6.109562, -6.055408, + -5.996362, -5.935081, -5.871264, + -6.328987, -6.301887, -6.276907, -6.244695, -6.205217, -6.156336, + -6.106204, -6.054013, -5.999475, -5.941885, + -6.286625, -6.282731, -6.267518, -6.242774, -6.20835, -6.168219, -6.125497, + -6.080946, -6.031807, -5.975547, + -6.193247, -6.187342, -6.183197, -6.175198, -6.155711, -6.131026, -6.09807, + -6.059319, -6.013174, -5.959172, + -6.058994, -6.055746, -6.05086, -6.046446, -6.038727, -6.018096, -5.98981, + -5.954857, -5.918507, -5.881165, + -6.561323, -6.499266, -6.438377, -6.375708, -6.310773, -6.244139, + -6.180099, -6.115896, -6.051152, -5.986147, + -6.845045, -6.786884, -6.725439, -6.661109, -6.601454, -6.540991, + -6.479097, -6.417562, -6.355603, -6.293946, + -7.102453, -7.045627, -6.989235, -6.932842, -6.874574, -6.816972, + -6.758873, -6.701811, -6.644189, -6.583754, + -7.351357, -7.301179, -7.248499, -7.19398, -7.142448, -7.089523, -7.035904, + -6.981055, -6.926404, -6.871471, + -7.598204, -7.549111, -7.497645, -7.447221, -7.397934, -7.350796, + -7.304009, -7.256029, -7.20568, -7.155462, + -7.82113, -7.78062, -7.739799, -7.69751, -7.652907, -7.609156, -7.56383, + -7.517503, -7.472572, -7.428266, + -8.020307, -7.985536, -7.955499, -7.923059, -7.885509, -7.841635, + -7.801394, -7.76416, -7.727195, -7.691062, + -8.203031, -8.185472, -8.161825, -8.130711, -8.093474, -8.056222, + -8.020072, -7.988377, -7.958443, -7.925798, + -8.343966, -8.324842, -8.308795, -8.291328, -8.267859, -8.240988, -8.21365, + -8.186219, -8.156057, -8.126547, + -8.44836, -8.431485, -8.415438, -8.404657, -8.391961, -8.373449, -8.350844, + -8.326425, -8.308137, -8.293455, + -7.418903, -7.380289, -7.343199, -7.30528, -7.266068, -7.225653, -7.186882, + -7.148771, -7.1104, -7.071782, + -7.886793, -7.851552, -7.814201, -7.774897, -7.739104, -7.703277, + -7.666594, -7.630272, -7.594023, -7.558314, + -8.341415, -8.308264, -8.274553, -8.240778, -8.205718, -8.171395, + -8.137189, -8.103981, -8.07057, -8.035443, + -8.796901, -8.767877, -8.73701, -8.704835, -8.674766, -8.644069, -8.61279, + -8.580782, -8.54905, -8.517601, + -9.253543, -9.225341, -9.195226, -9.165483, -9.137247, -9.111868, + -9.086359, -9.059315, -9.030994, -9.003092, + -9.699064, -9.677286, -9.654745, -9.63162, -9.607246, -9.582701, -9.557117, + -9.531214, -9.505648, -9.481256, + -10.12736, -10.10717, -10.09253, -10.07685, -10.05586, -10.03019, + -10.00788, -9.988334, -9.96856, -9.949014, + -10.53594, -10.52901, -10.51833, -10.50091, -10.47864, -10.45729, + -10.43758, -10.42123, -10.40631, -10.38915, + -10.90329, -10.89263, -10.88709, -10.88161, -10.87157, -10.86071, + -10.84752, -10.83391, -10.81815, -10.80181, + -11.248, -11.23781, -11.23062, -11.22926, -11.23034, -11.22393, -11.21365, + -11.20186, -11.19413, -11.18977, + -13.83239, -13.82074, -13.808, -13.79507, -13.78222, -13.76985, -13.75674, + -13.74387, -13.73124, -13.71895, + -14.99369, -14.98229, -14.97097, -14.96008, -14.94822, -14.9366, -14.92516, + -14.9145, -14.90532, -14.89437, + -16.12242, -16.11533, -16.10389, -16.09248, -16.08173, -16.06978, + -16.06084, -16.05067, -16.03975, -16.03102, + -17.2117, -17.20174, -17.1923, -17.18274, -17.17234, -17.16337, -17.15279, + -17.14514, -17.13551, -17.12395, + -18.23167, -18.22563, -18.21396, -18.20369, -18.19536, -18.19025, -18.1865, + -18.17599, -18.1694, -18.16037, + -19.16084, -19.15458, -19.15068, -19.14543, -19.14352, -19.13608, + -19.12984, -19.12499, -19.11406, -19.10741, + -19.96635, -19.96284, -19.96158, -19.96541, -19.9607, -19.95777, -19.95134, + -19.94964, -19.94552, -19.9427, + -20.64782, -20.64862, -20.65172, -20.64795, -20.64328, -20.63709, -20.6386, + -20.63893, -20.64199, -20.63851, + -21.1739, -21.17439, -21.17778, -21.18302, -21.18665, -21.19431, -21.1958, + -21.1981, -21.19873, -21.19386, + -21.59861, -21.59514, -21.59659, -21.60578, -21.61945, -21.62407, + -21.62483, -21.62446, -21.6267, -21.63452 ; + + bnds = 1, 2 ; + + lat_bnds = + -80, -79, + -79, -78, + -78, -77, + -77, -76, + -76, -75, + -75, -74, + -74, -73, + -73, -72, + -72, -71, + -71, -70 ; + + lon_bnds = + 12.5, 13.75, + 13.75, 15, + 15, 16.25, + 16.25, 17.5, + 17.5, 18.75, + 18.75, 20, + 20, 21.25, + 21.25, 22.5, + 22.5, 23.75, + 23.75, 25 ; + + time_bnds = + 9497, 9528 ; +} diff --git a/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_unmsk.case2.cdl b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_unmsk.case2.cdl new file mode 100644 index 0000000..346b074 --- /dev/null +++ b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_unmsk.case2.cdl @@ -0,0 +1,489 @@ +netcdf atmos_cmip.ua_unmsk { +dimensions: + bnds = 2 ; + lat = 10 ; + lon = 10 ; + plev19 = 19 ; + time = UNLIMITED ; // (1 currently) +variables: + double bnds(bnds) ; + bnds:long_name = "vertex number" ; + double lat(lat) ; + lat:long_name = "latitude" ; + lat:units = "degrees_N" ; + lat:axis = "Y" ; + lat:bounds = "lat_bnds" ; + double lat_bnds(lat, bnds) ; + lat_bnds:long_name = "latitude bounds" ; + lat_bnds:units = "degrees_N" ; + lat_bnds:axis = "Y" ; + double lon(lon) ; + lon:long_name = "longitude" ; + lon:units = "degrees_E" ; + lon:axis = "X" ; + lon:bounds = "lon_bnds" ; + double lon_bnds(lon, bnds) ; + lon_bnds:long_name = "longitude bounds" ; + lon_bnds:units = "degrees_E" ; + lon_bnds:axis = "X" ; + double plev19(plev19) ; + plev19:units = "Pa" ; + plev19:long_name = "pressure" ; + plev19:axis = "Z" ; + plev19:positive = "down" ; + double time(time) ; + time:units = "days since 1979-01-01 00:00:00" ; + time:long_name = "time" ; + time:axis = "T" ; + time:calendar_type = "JULIAN" ; + time:calendar = "julian" ; + time:bounds = "time_bnds" ; + double time_bnds(time, bnds) ; + time_bnds:units = "days since 1979-01-01 00:00:00" ; + time_bnds:long_name = "time axis boundaries" ; + float ua_unmsk(time, plev19, lat, lon) ; + ua_unmsk:_FillValue = 1.e+20f ; + ua_unmsk:missing_value = 1.e+20f ; + ua_unmsk:units = "m s-1" ; + ua_unmsk:long_name = "Eastward Wind" ; + ua_unmsk:cell_methods = "time: mean" ; + ua_unmsk:cell_measures = "area: area" ; + ua_unmsk:standard_name = "eastward_wind" ; + ua_unmsk:interp_method = "conserve_order2" ; + +// global attributes: + :title = "c96L65_am5f9d7r0_amip" ; + :associated_files = "area: 20050101.grid_spec.nc" ; + :grid_type = "regular" ; + :grid_tile = "N/A" ; + :code_release_version = "2024.05" ; + :git_hash = "5d306c05d9fe755cab04eedc8fd3de0d3c8355a0" ; + :creationtime = "Thu Jun 26 22:27:29 2025" ; + :hostname = "pp208" ; + :history = "Tue Jul 1 23:47:57 2025: ncks -d lat,10,19 -d lon,10,19 -d time,0,0 atmos_cmip.200501-200512.ua_unmsk.nc atmos_cmip.ua_unmsk.nc\n", + "fregrid --standard_dimension --input_mosaic C96_mosaic.nc --input_file 20050101.atmos_month_cmip --interp_method conserve_order2 --remap_file .fregrid_remap_file_288_by_180.nc --nlon 288 --nlat 180 --scalar_field tas,ts,psl,ps,uas,height10m,vas,sfcWind,hurs,height2m,huss,pr,prsn,prc,evspsbl,tauu,tauv,hfls,hfss,rlds,rlus,rsds,rsus,rsdscs,rsuscs,rldscs,rsdt,rsut,rlut,rlutcs,rsutcs,prw,clt,clwvi,clivi,rtmt,ccb,cct,ci,sci,ta_unmsk,ua_unmsk,va_unmsk,hus_unmsk,hur_unmsk,wap_unmsk,zg_unmsk,ap,b,ap_bnds,b_bnds,lev_bnds,utendnogw,utendogw,time_bnds --output_file out.nc" ; + :external_variables = "area" ; + :NCO = "netCDF Operators version 5.2.4 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco, Citation = 10.1016/j.envsoft.2008.03.004)" ; +data: + + bnds = 1, 2 ; + + lat = -79.5, -78.5, -77.5, -76.5, -75.5, -74.5, -73.5, -72.5, -71.5, -70.5 ; + + lat_bnds = + -80, -79, + -79, -78, + -78, -77, + -77, -76, + -76, -75, + -75, -74, + -74, -73, + -73, -72, + -72, -71, + -71, -70 ; + + lon = 13.125, 14.375, 15.625, 16.875, 18.125, 19.375, 20.625, 21.875, + 23.125, 24.375 ; + + lon_bnds = + 12.5, 13.75, + 13.75, 15, + 15, 16.25, + 16.25, 17.5, + 17.5, 18.75, + 18.75, 20, + 20, 21.25, + 21.25, 22.5, + 22.5, 23.75, + 23.75, 25 ; + + plev19 = 100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, + 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000, 500, 100 ; + + time = 9512.5 ; + + time_bnds = + 9497, 9528 ; + + ua_unmsk = + -0.3975942, -0.2708012, -0.1832284, -0.09187625, 0.001559797, 0.07204625, + 0.0783459, 0.09246785, 0.1097215, 0.1203732, + -0.5745555, -0.5160208, -0.4475087, -0.3612338, -0.3011235, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.151411, -2.133208, -2.145542, -2.198483, -2.335929, -2.401734, + -2.460874, -2.553266, -2.576532, -2.571083, + -2.586592, -2.56783, -2.469594, -2.440526, -2.463234, -2.546937, -2.664745, + -2.762299, -2.825696, -2.793565, + -2.590438, -2.542438, -2.476797, -2.352731, -2.229772, -2.188571, + -2.246655, -2.410638, -2.534509, -2.572949, + -1.956838, -1.938947, -1.808724, -1.627707, -1.515042, -1.380297, + -1.431068, -1.480767, -1.527824, -1.59102, + -0.5315368, -0.5231183, -0.4564048, -0.3944708, -0.1738839, -0.07352839, + 0.01055507, 0.1166717, 0.1312627, 0.2858787, + -0.3975942, -0.2708012, -0.1832284, -0.09187625, 0.001559797, 0.07204625, + 0.0783459, 0.09246785, 0.1097215, 0.1203732, + -0.5745555, -0.5160208, -0.4475087, -0.3612338, -0.3011235, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.151411, -2.133208, -2.145542, -2.198483, -2.335929, -2.401734, + -2.460874, -2.553266, -2.576532, -2.571083, + -2.586592, -2.56783, -2.469594, -2.440526, -2.463234, -2.546937, -2.664745, + -2.76217, -2.823994, -2.793565, + -2.56729, -2.51979, -2.476788, -2.342977, -2.180678, -2.173315, -2.202794, + -2.382988, -2.51635, -2.56469, + -2.164203, -2.002625, -1.769979, -1.522183, -1.529329, -1.788184, + -1.570996, -1.601109, -2.148522, -2.238381, + -1.765279, -1.833698, -1.684174, -1.374214, -1.259788, -1.473578, + -1.366698, -1.105641, -1.151468, -1.052101, + -0.3975942, -0.2708012, -0.1832284, -0.09187625, 0.001559797, 0.07204625, + 0.0783459, 0.09246785, 0.1097215, 0.1203732, + -0.5745555, -0.5160208, -0.4475087, -0.3612338, -0.3011235, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.151411, -2.133208, -2.145542, -2.198483, -2.335929, -2.401734, + -2.460874, -2.553266, -2.576532, -2.571083, + -2.586592, -2.559288, -2.461631, -2.440526, -2.46086, -2.4672, -2.609189, + -2.751828, -2.816938, -2.793565, + -2.410764, -2.404498, -2.467111, -2.314096, -1.887833, -2.113094, + -2.547507, -2.529567, -2.430087, -2.455852, + -2.803633, -2.214195, -2.758755, -3.378312, -3.186033, -3.65111, -4.398364, + -4.396877, -4.207345, -3.572534, + -5.248218, -4.912538, -4.553966, -4.296717, -4.063207, -4.280607, + -4.150732, -3.721014, -3.682456, -3.493971, + -0.4638011, -0.2794998, -0.1847423, -0.1013127, -0.01874243, 0.05095983, + 0.07407396, 0.09246785, 0.1097215, 0.1203732, + -0.5747783, -0.5167099, -0.4489113, -0.3624935, -0.3011871, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.046627, -2.075339, -2.133872, -2.192768, -2.29642, -2.385374, -2.421421, + -2.375845, -2.522865, -2.570514, + -2.329157, -2.661558, -2.550557, -2.222389, -2.452235, -2.64684, -2.825807, + -3.021084, -2.930932, -2.340356, + -4.449573, -5.811647, -6.238008, -5.904782, -5.536724, -5.204103, + -5.139505, -6.584569, -7.315174, -6.587632, + -7.759037, -7.923704, -7.657747, -7.342247, -7.189469, -6.957999, + -6.869977, -6.911683, -6.890206, -6.571945, + -5.766199, -5.629292, -5.500316, -5.457587, -5.137841, -4.946566, + -4.747363, -4.453594, -4.333026, -4.211693, + 1.063227, 1.156564, 1.210127, 1.265683, 1.325263, 1.37135, 1.379399, + 1.395321, 1.408047, 1.413615, + 0.8057297, 0.823011, 0.8517545, 0.8839636, 0.8734952, 0.8716547, 0.8792083, + 0.8959213, 0.933274, 0.9308779, + 0.3776965, 0.3511574, 0.3064211, 0.2771505, 0.2723292, 0.26952, 0.2818037, + 0.2319987, 0.165376, 0.1122594, + -0.2686788, -0.3904975, -0.4879307, -0.5713966, -0.6487127, -0.7579689, + -0.879656, -1.019151, -1.160378, -1.290107, + -1.44353, -1.544772, -1.67499, -1.812897, -1.98716, -2.274205, -2.556359, + -2.771371, -2.97445, -3.201544, + -3.11445, -3.268162, -3.425929, -3.644483, -3.970347, -4.194717, -4.426955, + -4.785159, -4.986393, -5.121875, + -4.842557, -4.942987, -5.010421, -5.153152, -5.254729, -5.326348, + -5.648958, -6.018217, -6.451276, -6.772595, + -6.110239, -6.028393, -5.95482, -5.914704, -5.883089, -5.694447, -5.721324, + -5.85588, -6.038979, -6.222065, + -6.000411, -6.046391, -5.883338, -5.710005, -5.577016, -5.360312, + -5.450391, -5.419651, -5.09429, -4.927251, + -4.122381, -4.0073, -3.920688, -3.890532, -3.619308, -3.348911, -3.164834, + -3.032901, -2.999816, -2.981941, + 1.021295, 1.058732, 1.055263, 1.055828, 1.06735, 1.085502, 1.072789, + 1.066372, 1.055719, 1.03737, + 0.5644462, 0.5258871, 0.505933, 0.5047766, 0.4665931, 0.4326979, 0.4054725, + 0.3554713, 0.3202047, 0.2826239, + -0.09909091, -0.1695052, -0.2423104, -0.308107, -0.3618009, -0.4362217, + -0.5038798, -0.5690737, -0.6322247, -0.6912587, + -0.8452172, -0.9907208, -1.12432, -1.275977, -1.407145, -1.514558, + -1.611202, -1.70489, -1.787341, -1.840466, + -1.660944, -1.836267, -2.043581, -2.208486, -2.365314, -2.544751, + -2.701359, -2.791034, -2.877407, -2.950873, + -2.49663, -2.691726, -2.876794, -3.089699, -3.315934, -3.422023, -3.487426, + -3.497974, -3.590583, -3.699841, + -3.409052, -3.454748, -3.579855, -3.723231, -3.777681, -3.849489, + -3.761075, -3.758978, -3.830392, -3.977781, + -4.332948, -4.30478, -4.268668, -4.142052, -4.038466, -3.906201, -3.853595, + -3.97838, -4.061214, -3.96847, + -4.037989, -3.923016, -3.725387, -3.504999, -3.335768, -3.048127, + -2.881483, -2.777315, -2.57696, -2.525698, + -2.675588, -2.521989, -2.357147, -2.217182, -1.92482, -1.59737, -1.311503, + -1.052147, -0.8554193, -0.7043628, + 0.868393, 0.8096983, 0.7342755, 0.6706711, 0.6231919, 0.5861326, 0.5235301, + 0.4832453, 0.4458269, 0.403123, + 0.3249699, 0.203376, 0.09710471, 0.005840228, -0.09959703, -0.186581, + -0.2659973, -0.3638777, -0.4377598, -0.5044919, + -0.4682975, -0.6448929, -0.7865676, -0.9014125, -1.010695, -1.141248, + -1.26495, -1.354475, -1.432743, -1.508723, + -1.302762, -1.487646, -1.667504, -1.870815, -2.038251, -2.165048, + -2.268736, -2.35599, -2.422225, -2.462221, + -1.897165, -2.089815, -2.293899, -2.467101, -2.626223, -2.795213, + -2.938092, -3.009527, -3.077491, -3.088423, + -2.338803, -2.44815, -2.576407, -2.714023, -2.881914, -2.990129, -3.088588, + -3.162073, -3.200975, -3.280721, + -2.629382, -2.617197, -2.644625, -2.714817, -2.728057, -2.764721, + -2.755495, -2.804715, -2.900975, -3.060241, + -2.806314, -2.699975, -2.604603, -2.467385, -2.316504, -2.181494, + -2.104434, -2.068235, -2.089983, -2.092961, + -2.350076, -2.191863, -2.015235, -1.808842, -1.591449, -1.298141, + -1.055869, -0.8611979, -0.6329716, -0.4672858, + -1.417739, -1.253469, -1.085745, -0.9032833, -0.6274295, -0.3408952, + -0.04521877, 0.2827266, 0.5437686, 0.7892594, + 0.08603396, -0.01244956, -0.1302382, -0.230692, -0.3224648, -0.4244072, + -0.5279509, -0.5862728, -0.6336403, -0.6853408, + -0.5083337, -0.6752201, -0.8401579, -1.021567, -1.198536, -1.330774, + -1.454516, -1.586487, -1.691136, -1.77504, + -1.209975, -1.465892, -1.672402, -1.836556, -2.003343, -2.175733, + -2.340954, -2.462788, -2.570125, -2.671328, + -1.835453, -2.040931, -2.250755, -2.488194, -2.693811, -2.848905, + -2.973454, -3.076011, -3.155657, -3.221975, + -2.087997, -2.262499, -2.42809, -2.583258, -2.728339, -2.874663, -3.012026, + -3.105144, -3.219588, -3.292633, + -2.219754, -2.239856, -2.291842, -2.338402, -2.416109, -2.477032, + -2.551712, -2.625169, -2.696625, -2.812562, + -2.214184, -2.120897, -2.02604, -1.978979, -1.914884, -1.861667, -1.797479, + -1.808812, -1.854385, -1.959785, + -2.145867, -1.937978, -1.761547, -1.585332, -1.406883, -1.215052, + -1.064301, -0.9400889, -0.883372, -0.8476374, + -1.6123, -1.449645, -1.277987, -1.08922, -0.8639328, -0.5785105, + -0.3019016, -0.06757384, 0.1769251, 0.3469639, + -0.4829444, -0.3573219, -0.2495783, -0.1331687, 0.03723535, 0.2394148, + 0.4937723, 0.8273596, 1.081453, 1.332125, + -0.8436456, -0.9486177, -1.059377, -1.153291, -1.241334, -1.344937, + -1.442711, -1.494026, -1.531425, -1.569082, + -1.363601, -1.517111, -1.673566, -1.853703, -2.024385, -2.153884, + -2.277348, -2.399244, -2.49864, -2.581057, + -1.900008, -2.121459, -2.298829, -2.446108, -2.600936, -2.75574, -2.91209, + -3.041326, -3.159974, -3.291958, + -2.332246, -2.485257, -2.649413, -2.831298, -2.996419, -3.138222, + -3.260611, -3.390049, -3.512205, -3.606771, + -2.484051, -2.582164, -2.656566, -2.747351, -2.843552, -2.953647, + -3.080211, -3.176885, -3.30392, -3.404524, + -2.524805, -2.482363, -2.478343, -2.466771, -2.482541, -2.493074, + -2.524661, -2.568022, -2.622594, -2.706606, + -2.361249, -2.253306, -2.134065, -2.054415, -1.964403, -1.889408, + -1.791761, -1.750293, -1.718029, -1.732389, + -2.014955, -1.839144, -1.68769, -1.535044, -1.384408, -1.211162, -1.050033, + -0.8907735, -0.7928692, -0.7119153, + -1.236629, -1.093662, -0.9508283, -0.8085872, -0.6442373, -0.4315634, + -0.1945382, -0.002699569, 0.1975364, 0.3391424, + -0.113028, 0.03701114, 0.1509672, 0.244096, 0.3440107, 0.4850455, 0.659207, + 0.8812482, 1.058411, 1.222257, + -1.854996, -1.890166, -1.915619, -1.932104, -1.946811, -1.971547, + -1.988159, -1.97958, -1.964028, -1.948233, + -2.224256, -2.271724, -2.324106, -2.393641, -2.449779, -2.481474, + -2.509733, -2.527462, -2.532758, -2.527077, + -2.520393, -2.60658, -2.663467, -2.701849, -2.744738, -2.780796, -2.817967, + -2.841355, -2.859165, -2.87995, + -2.696981, -2.736231, -2.781672, -2.833747, -2.878667, -2.9107, -2.931204, + -2.952544, -2.963497, -2.964039, + -2.669573, -2.676523, -2.663651, -2.663748, -2.672803, -2.703568, -2.73891, + -2.744817, -2.765156, -2.767472, + -2.476536, -2.429364, -2.405869, -2.377028, -2.36033, -2.332005, -2.306961, + -2.277795, -2.2701, -2.287573, + -2.063781, -1.986485, -1.90999, -1.85632, -1.792651, -1.720523, -1.636689, + -1.607912, -1.594598, -1.61015, + -1.519658, -1.429246, -1.343379, -1.238522, -1.116604, -1.015946, + -0.9358807, -0.8798366, -0.8500427, -0.8273263, + -0.7674065, -0.6452081, -0.5515807, -0.4693587, -0.3846785, -0.3116637, + -0.2242767, -0.1593899, -0.1116388, -0.06896764, + -0.01183324, 0.1091698, 0.2188423, 0.312474, 0.3854575, 0.4438081, + 0.5140151, 0.5824342, 0.6187019, 0.6379458, + -2.684902, -2.610719, -2.535948, -2.460751, -2.387049, -2.321111, + -2.254766, -2.182338, -2.107151, -2.031494, + -2.820069, -2.740268, -2.659949, -2.584899, -2.516798, -2.444206, + -2.371544, -2.295798, -2.221301, -2.142587, + -2.920424, -2.834318, -2.745224, -2.660668, -2.577662, -2.494831, -2.4213, + -2.345643, -2.268257, -2.192751, + -2.919983, -2.838565, -2.757891, -2.677184, -2.599998, -2.519447, + -2.432564, -2.343707, -2.255901, -2.170567, + -2.795936, -2.718457, -2.634572, -2.554108, -2.473378, -2.391261, + -2.312509, -2.233216, -2.158018, -2.077407, + -2.562569, -2.481418, -2.403707, -2.320077, -2.236581, -2.159199, + -2.083839, -2.011575, -1.949767, -1.892647, + -2.250703, -2.162687, -2.078802, -2.00374, -1.924842, -1.843397, -1.762613, + -1.697863, -1.633513, -1.583002, + -1.825546, -1.773501, -1.720045, -1.646263, -1.569409, -1.491826, + -1.415384, -1.344221, -1.289576, -1.235646, + -1.16891, -1.113447, -1.082653, -1.06613, -1.038724, -1.014261, -0.971841, + -0.9325086, -0.8841423, -0.8362257, + -0.4466377, -0.3950162, -0.3640113, -0.3467484, -0.3497896, -0.3435763, + -0.3285452, -0.3093777, -0.3015329, -0.2992152, + -3.47265, -3.349887, -3.225127, -3.099491, -2.975098, -2.858005, -2.742823, + -2.620348, -2.497191, -2.375009, + -3.669024, -3.520918, -3.372771, -3.233594, -3.098513, -2.952549, + -2.805939, -2.665119, -2.521485, -2.376583, + -3.875113, -3.713779, -3.561024, -3.408706, -3.2558, -3.100774, -2.95319, + -2.807744, -2.659667, -2.510706, + -3.950386, -3.795719, -3.641412, -3.475792, -3.323338, -3.181724, + -3.039761, -2.904638, -2.775108, -2.650002, + -3.947389, -3.783504, -3.606038, -3.452417, -3.30665, -3.171729, -3.049297, + -2.93359, -2.822966, -2.726708, + -3.906445, -3.746139, -3.594896, -3.442577, -3.297046, -3.1648, -3.036906, + -2.916452, -2.812173, -2.712864, + -3.83857, -3.71836, -3.602963, -3.481781, -3.346611, -3.205612, -3.061509, + -2.923938, -2.794568, -2.691266, + -3.559861, -3.530004, -3.478284, -3.396861, -3.302307, -3.188716, + -3.069149, -2.94336, -2.817003, -2.663864, + -2.915129, -2.919575, -2.929287, -2.93514, -2.908199, -2.866826, -2.797559, + -2.709057, -2.578434, -2.428774, + -2.010599, -2.042224, -2.077676, -2.104624, -2.122853, -2.120454, + -2.097497, -2.047853, -1.992431, -1.92157, + -4.22112, -4.07559, -3.922843, -3.768222, -3.614719, -3.473573, -3.340809, + -3.200793, -3.061464, -2.926893, + -4.502892, -4.344226, -4.183433, -4.026276, -3.878217, -3.729609, -3.58069, + -3.431968, -3.287462, -3.145941, + -4.631217, -4.473382, -4.315218, -4.164486, -4.013389, -3.85793, -3.710944, + -3.570176, -3.429156, -3.28708, + -4.626989, -4.476557, -4.326241, -4.174338, -4.032887, -3.895635, + -3.756263, -3.619544, -3.485802, -3.356973, + -4.556738, -4.395508, -4.235921, -4.096162, -3.958964, -3.832275, + -3.715655, -3.600988, -3.489401, -3.387974, + -4.468916, -4.302645, -4.15163, -4.001896, -3.865624, -3.743882, -3.626797, + -3.521506, -3.429111, -3.338457, + -4.44977, -4.292527, -4.144361, -4.001679, -3.853956, -3.707088, -3.567008, + -3.438192, -3.317925, -3.220261, + -4.439157, -4.329401, -4.202333, -4.056956, -3.905393, -3.736642, + -3.568599, -3.403389, -3.260043, -3.114572, + -4.203069, -4.12499, -4.031964, -3.926179, -3.793952, -3.63649, -3.470912, + -3.312045, -3.139288, -2.968029, + -3.549062, -3.518938, -3.466148, -3.407319, -3.308811, -3.204609, + -3.090285, -2.963688, -2.832902, -2.702063, + -4.634931, -4.510322, -4.387384, -4.262967, -4.13814, -4.019274, -3.907151, + -3.793168, -3.677498, -3.562162, + -4.812496, -4.685526, -4.557202, -4.432652, -4.315062, -4.196854, + -4.077685, -3.961596, -3.849445, -3.737238, + -4.926235, -4.801914, -4.682349, -4.564368, -4.445381, -4.328474, + -4.217326, -4.105275, -3.991226, -3.87654, + -4.974308, -4.86126, -4.746056, -4.629657, -4.521437, -4.412365, -4.300677, + -4.186651, -4.075935, -3.968378, + -4.938999, -4.831478, -4.722439, -4.619403, -4.516479, -4.414182, + -4.313823, -4.215149, -4.114635, -4.01406, + -4.802555, -4.709178, -4.621257, -4.533046, -4.446228, -4.362994, + -4.278905, -4.198927, -4.114691, -4.029364, + -4.646967, -4.549815, -4.467683, -4.393793, -4.318109, -4.249639, + -4.184724, -4.118573, -4.056467, -3.996501, + -4.553621, -4.461698, -4.370646, -4.279413, -4.197869, -4.108099, + -4.025449, -3.951957, -3.882595, -3.806807, + -4.450593, -4.350024, -4.254059, -4.160119, -4.047391, -3.930037, + -3.814608, -3.705483, -3.598191, -3.498837, + -4.176404, -4.088185, -3.99167, -3.888712, -3.763858, -3.636919, -3.503574, + -3.371032, -3.253147, -3.15005, + -5.305916, -5.228198, -5.149081, -5.066242, -4.979733, -4.891483, + -4.805131, -4.715399, -4.625538, -4.536609, + -5.409827, -5.338651, -5.264568, -5.190886, -5.118539, -5.040178, + -4.958767, -4.879787, -4.798156, -4.714354, + -5.436405, -5.377537, -5.321352, -5.260819, -5.19785, -5.135309, -5.069129, + -4.99902, -4.925947, -4.848398, + -5.425586, -5.371675, -5.315626, -5.258965, -5.205391, -5.147702, + -5.090356, -5.029489, -4.966897, -4.903516, + -5.421552, -5.364684, -5.306411, -5.247279, -5.189979, -5.13738, -5.082496, + -5.023454, -4.961823, -4.898187, + -5.399646, -5.353481, -5.305019, -5.255711, -5.203397, -5.146911, + -5.086962, -5.023842, -4.958927, -4.891607, + -5.324206, -5.285987, -5.253324, -5.216431, -5.17214, -5.119621, -5.066675, + -5.011673, -4.952978, -4.889044, + -5.201594, -5.185238, -5.161349, -5.129875, -5.091774, -5.047886, + -5.002157, -4.954802, -4.903096, -4.845263, + -5.027672, -5.009829, -4.996872, -4.982873, -4.957479, -4.928816, + -4.894786, -4.855186, -4.809749, -4.755376, + -4.800905, -4.790343, -4.778637, -4.76788, -4.754467, -4.730603, -4.70276, + -4.672376, -4.637187, -4.600635, + -5.877733, -5.806942, -5.737911, -5.665781, -5.589697, -5.508978, + -5.428902, -5.346344, -5.262498, -5.177478, + -5.999748, -5.93775, -5.871769, -5.80268, -5.736236, -5.664495, -5.589656, + -5.515277, -5.436458, -5.355067, + -6.105768, -6.050148, -5.99489, -5.934047, -5.869964, -5.807178, -5.740132, + -5.669778, -5.59723, -5.518231, + -6.204219, -6.156556, -6.104576, -6.049919, -5.996562, -5.93629, -5.873829, + -5.805651, -5.734739, -5.662208, + -6.283307, -6.238613, -6.188774, -6.136417, -6.083857, -6.031115, + -5.974864, -5.914575, -5.849953, -5.782243, + -6.327734, -6.29085, -6.250577, -6.207967, -6.160203, -6.109562, -6.055408, + -5.996362, -5.935081, -5.871264, + -6.328987, -6.301887, -6.276907, -6.244695, -6.205217, -6.156336, + -6.106204, -6.054013, -5.999475, -5.941885, + -6.286625, -6.282731, -6.267518, -6.242774, -6.20835, -6.168219, -6.125497, + -6.080946, -6.031807, -5.975547, + -6.193247, -6.187342, -6.183197, -6.175198, -6.155711, -6.131026, -6.09807, + -6.059319, -6.013174, -5.959172, + -6.058994, -6.055746, -6.05086, -6.046446, -6.038727, -6.018096, -5.98981, + -5.954857, -5.918507, -5.881165, + -6.561323, -6.499266, -6.438377, -6.375708, -6.310773, -6.244139, + -6.180099, -6.115896, -6.051152, -5.986147, + -6.845045, -6.786884, -6.725439, -6.661109, -6.601454, -6.540991, + -6.479097, -6.417562, -6.355603, -6.293946, + -7.102453, -7.045627, -6.989235, -6.932842, -6.874574, -6.816972, + -6.758873, -6.701811, -6.644189, -6.583754, + -7.351357, -7.301179, -7.248499, -7.19398, -7.142448, -7.089523, -7.035904, + -6.981055, -6.926404, -6.871471, + -7.598204, -7.549111, -7.497645, -7.447221, -7.397934, -7.350796, + -7.304009, -7.256029, -7.20568, -7.155462, + -7.82113, -7.78062, -7.739799, -7.69751, -7.652907, -7.609156, -7.56383, + -7.517503, -7.472572, -7.428266, + -8.020307, -7.985536, -7.955499, -7.923059, -7.885509, -7.841635, + -7.801394, -7.76416, -7.727195, -7.691062, + -8.203031, -8.185472, -8.161825, -8.130711, -8.093474, -8.056222, + -8.020072, -7.988377, -7.958443, -7.925798, + -8.343966, -8.324842, -8.308795, -8.291328, -8.267859, -8.240988, -8.21365, + -8.186219, -8.156057, -8.126547, + -8.44836, -8.431485, -8.415438, -8.404657, -8.391961, -8.373449, -8.350844, + -8.326425, -8.308137, -8.293455, + -7.418903, -7.380289, -7.343199, -7.30528, -7.266068, -7.225653, -7.186882, + -7.148771, -7.1104, -7.071782, + -7.886793, -7.851552, -7.814201, -7.774897, -7.739104, -7.703277, + -7.666594, -7.630272, -7.594023, -7.558314, + -8.341415, -8.308264, -8.274553, -8.240778, -8.205718, -8.171395, + -8.137189, -8.103981, -8.07057, -8.035443, + -8.796901, -8.767877, -8.73701, -8.704835, -8.674766, -8.644069, -8.61279, + -8.580782, -8.54905, -8.517601, + -9.253543, -9.225341, -9.195226, -9.165483, -9.137247, -9.111868, + -9.086359, -9.059315, -9.030994, -9.003092, + -9.699064, -9.677286, -9.654745, -9.63162, -9.607246, -9.582701, -9.557117, + -9.531214, -9.505648, -9.481256, + -10.12736, -10.10717, -10.09253, -10.07685, -10.05586, -10.03019, + -10.00788, -9.988334, -9.96856, -9.949014, + -10.53594, -10.52901, -10.51833, -10.50091, -10.47864, -10.45729, + -10.43758, -10.42123, -10.40631, -10.38915, + -10.90329, -10.89263, -10.88709, -10.88161, -10.87157, -10.86071, + -10.84752, -10.83391, -10.81815, -10.80181, + -11.248, -11.23781, -11.23062, -11.22926, -11.23034, -11.22393, -11.21365, + -11.20186, -11.19413, -11.18977, + -13.83239, -13.82074, -13.808, -13.79507, -13.78222, -13.76985, -13.75674, + -13.74387, -13.73124, -13.71895, + -14.99369, -14.98229, -14.97097, -14.96008, -14.94822, -14.9366, -14.92516, + -14.9145, -14.90532, -14.89437, + -16.12242, -16.11533, -16.10389, -16.09248, -16.08173, -16.06978, + -16.06084, -16.05067, -16.03975, -16.03102, + -17.2117, -17.20174, -17.1923, -17.18274, -17.17234, -17.16337, -17.15279, + -17.14514, -17.13551, -17.12395, + -18.23167, -18.22563, -18.21396, -18.20369, -18.19536, -18.19025, -18.1865, + -18.17599, -18.1694, -18.16037, + -19.16084, -19.15458, -19.15068, -19.14543, -19.14352, -19.13608, + -19.12984, -19.12499, -19.11406, -19.10741, + -19.96635, -19.96284, -19.96158, -19.96541, -19.9607, -19.95777, -19.95134, + -19.94964, -19.94552, -19.9427, + -20.64782, -20.64862, -20.65172, -20.64795, -20.64328, -20.63709, -20.6386, + -20.63893, -20.64199, -20.63851, + -21.1739, -21.17439, -21.17778, -21.18302, -21.18665, -21.19431, -21.1958, + -21.1981, -21.19873, -21.19386, + -21.59861, -21.59514, -21.59659, -21.60578, -21.61945, -21.62407, + -21.62483, -21.62446, -21.6267, -21.63452 ; +} diff --git a/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_unmsk.cdl b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_unmsk.cdl new file mode 100644 index 0000000..def2bc5 --- /dev/null +++ b/fremorizer/tests/test_files/reduced_ascii_files/atmos_cmip.ua_unmsk.cdl @@ -0,0 +1,490 @@ +netcdf atmos_cmip.ua_unmsk { +dimensions: + bnds = 2 ; + lat = 10 ; + lon = 10 ; + plev19 = 19 ; + time = UNLIMITED ; // (1 currently) +variables: + double bnds(bnds) ; + bnds:long_name = "vertex number" ; + double lat(lat) ; + lat:long_name = "latitude" ; + lat:units = "degrees_N" ; + lat:axis = "Y" ; + lat:bounds = "lat_bnds" ; + double lat_bnds(lat, bnds) ; + lat_bnds:long_name = "latitude bounds" ; + lat_bnds:units = "degrees_N" ; + lat_bnds:axis = "Y" ; + double lon(lon) ; + lon:long_name = "longitude" ; + lon:units = "degrees_E" ; + lon:axis = "X" ; + lon:bounds = "lon_bnds" ; + double lon_bnds(lon, bnds) ; + lon_bnds:long_name = "longitude bounds" ; + lon_bnds:units = "degrees_E" ; + lon_bnds:axis = "X" ; + double plev19(plev19) ; + plev19:units = "Pa" ; + plev19:long_name = "pressure" ; + plev19:axis = "Z" ; + plev19:positive = "down" ; + double time(time) ; + time:units = "days since 1979-01-01 00:00:00" ; + time:long_name = "time" ; + time:axis = "T" ; + time:calendar_type = "JULIAN" ; + time:calendar = "julian" ; + time:bounds = "time_bnds" ; + double time_bnds(time, bnds) ; + time_bnds:units = "days since 1979-01-01 00:00:00" ; + time_bnds:long_name = "time axis boundaries" ; + float ua_unmsk(time, plev19, lat, lon) ; + ua_unmsk:_FillValue = 1.e+20f ; + ua_unmsk:missing_value = 1.e+20f ; + ua_unmsk:units = "m s-1" ; + ua_unmsk:long_name = "Eastward Wind" ; + ua_unmsk:cell_methods = "time: mean" ; + ua_unmsk:cell_measures = "area: area" ; + ua_unmsk:standard_name = "eastward_wind" ; + ua_unmsk:interp_method = "conserve_order2" ; + ua_unmsk:pressure_mask = "False" ; + +// global attributes: + :title = "c96L65_am5f9d7r0_amip" ; + :associated_files = "area: 20050101.grid_spec.nc" ; + :grid_type = "regular" ; + :grid_tile = "N/A" ; + :code_release_version = "2024.05" ; + :git_hash = "5d306c05d9fe755cab04eedc8fd3de0d3c8355a0" ; + :creationtime = "Thu Jun 26 22:27:29 2025" ; + :hostname = "pp208" ; + :history = "Tue Jul 1 23:47:57 2025: ncks -d lat,10,19 -d lon,10,19 -d time,0,0 atmos_cmip.200501-200512.ua_unmsk.nc atmos_cmip.ua_unmsk.nc\n", + "fregrid --standard_dimension --input_mosaic C96_mosaic.nc --input_file 20050101.atmos_month_cmip --interp_method conserve_order2 --remap_file .fregrid_remap_file_288_by_180.nc --nlon 288 --nlat 180 --scalar_field tas,ts,psl,ps,uas,height10m,vas,sfcWind,hurs,height2m,huss,pr,prsn,prc,evspsbl,tauu,tauv,hfls,hfss,rlds,rlus,rsds,rsus,rsdscs,rsuscs,rldscs,rsdt,rsut,rlut,rlutcs,rsutcs,prw,clt,clwvi,clivi,rtmt,ccb,cct,ci,sci,ta_unmsk,ua_unmsk,va_unmsk,hus_unmsk,hur_unmsk,wap_unmsk,zg_unmsk,ap,b,ap_bnds,b_bnds,lev_bnds,utendnogw,utendogw,time_bnds --output_file out.nc" ; + :external_variables = "area" ; + :NCO = "netCDF Operators version 5.2.4 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco, Citation = 10.1016/j.envsoft.2008.03.004)" ; +data: + + bnds = 1, 2 ; + + lat = -79.5, -78.5, -77.5, -76.5, -75.5, -74.5, -73.5, -72.5, -71.5, -70.5 ; + + lat_bnds = + -80, -79, + -79, -78, + -78, -77, + -77, -76, + -76, -75, + -75, -74, + -74, -73, + -73, -72, + -72, -71, + -71, -70 ; + + lon = 13.125, 14.375, 15.625, 16.875, 18.125, 19.375, 20.625, 21.875, + 23.125, 24.375 ; + + lon_bnds = + 12.5, 13.75, + 13.75, 15, + 15, 16.25, + 16.25, 17.5, + 17.5, 18.75, + 18.75, 20, + 20, 21.25, + 21.25, 22.5, + 22.5, 23.75, + 23.75, 25 ; + + plev19 = 100000, 92500, 85000, 70000, 60000, 50000, 40000, 30000, 25000, + 20000, 15000, 10000, 7000, 5000, 3000, 2000, 1000, 500, 100 ; + + time = 9512.5 ; + + time_bnds = + 9497, 9528 ; + + ua_unmsk = + -0.3975942, -0.2708012, -0.1832284, -0.09187625, 0.001559797, 0.07204625, + 0.0783459, 0.09246785, 0.1097215, 0.1203732, + -0.5745555, -0.5160208, -0.4475087, -0.3612338, -0.3011235, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.151411, -2.133208, -2.145542, -2.198483, -2.335929, -2.401734, + -2.460874, -2.553266, -2.576532, -2.571083, + -2.586592, -2.56783, -2.469594, -2.440526, -2.463234, -2.546937, -2.664745, + -2.762299, -2.825696, -2.793565, + -2.590438, -2.542438, -2.476797, -2.352731, -2.229772, -2.188571, + -2.246655, -2.410638, -2.534509, -2.572949, + -1.956838, -1.938947, -1.808724, -1.627707, -1.515042, -1.380297, + -1.431068, -1.480767, -1.527824, -1.59102, + -0.5315368, -0.5231183, -0.4564048, -0.3944708, -0.1738839, -0.07352839, + 0.01055507, 0.1166717, 0.1312627, 0.2858787, + -0.3975942, -0.2708012, -0.1832284, -0.09187625, 0.001559797, 0.07204625, + 0.0783459, 0.09246785, 0.1097215, 0.1203732, + -0.5745555, -0.5160208, -0.4475087, -0.3612338, -0.3011235, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.151411, -2.133208, -2.145542, -2.198483, -2.335929, -2.401734, + -2.460874, -2.553266, -2.576532, -2.571083, + -2.586592, -2.56783, -2.469594, -2.440526, -2.463234, -2.546937, -2.664745, + -2.76217, -2.823994, -2.793565, + -2.56729, -2.51979, -2.476788, -2.342977, -2.180678, -2.173315, -2.202794, + -2.382988, -2.51635, -2.56469, + -2.164203, -2.002625, -1.769979, -1.522183, -1.529329, -1.788184, + -1.570996, -1.601109, -2.148522, -2.238381, + -1.765279, -1.833698, -1.684174, -1.374214, -1.259788, -1.473578, + -1.366698, -1.105641, -1.151468, -1.052101, + -0.3975942, -0.2708012, -0.1832284, -0.09187625, 0.001559797, 0.07204625, + 0.0783459, 0.09246785, 0.1097215, 0.1203732, + -0.5745555, -0.5160208, -0.4475087, -0.3612338, -0.3011235, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.151411, -2.133208, -2.145542, -2.198483, -2.335929, -2.401734, + -2.460874, -2.553266, -2.576532, -2.571083, + -2.586592, -2.559288, -2.461631, -2.440526, -2.46086, -2.4672, -2.609189, + -2.751828, -2.816938, -2.793565, + -2.410764, -2.404498, -2.467111, -2.314096, -1.887833, -2.113094, + -2.547507, -2.529567, -2.430087, -2.455852, + -2.803633, -2.214195, -2.758755, -3.378312, -3.186033, -3.65111, -4.398364, + -4.396877, -4.207345, -3.572534, + -5.248218, -4.912538, -4.553966, -4.296717, -4.063207, -4.280607, + -4.150732, -3.721014, -3.682456, -3.493971, + -0.4638011, -0.2794998, -0.1847423, -0.1013127, -0.01874243, 0.05095983, + 0.07407396, 0.09246785, 0.1097215, 0.1203732, + -0.5747783, -0.5167099, -0.4489113, -0.3624935, -0.3011871, -0.2559909, + -0.2050162, -0.1434478, -0.08277573, -0.06627619, + -0.6454812, -0.6330528, -0.6397436, -0.6381702, -0.6238125, -0.5942059, + -0.5442353, -0.5295258, -0.5248909, -0.527373, + -0.8545123, -0.8994937, -0.9365253, -1.021321, -1.10686, -1.154233, + -1.184383, -1.207665, -1.235654, -1.261622, + -1.426775, -1.437999, -1.512454, -1.57849, -1.644708, -1.758953, -1.863996, + -1.918497, -1.973612, -2.010429, + -2.046627, -2.075339, -2.133872, -2.192768, -2.29642, -2.385374, -2.421421, + -2.375845, -2.522865, -2.570514, + -2.329157, -2.661558, -2.550557, -2.222389, -2.452235, -2.64684, -2.825807, + -3.021084, -2.930932, -2.340356, + -4.449573, -5.811647, -6.238008, -5.904782, -5.536724, -5.204103, + -5.139505, -6.584569, -7.315174, -6.587632, + -7.759037, -7.923704, -7.657747, -7.342247, -7.189469, -6.957999, + -6.869977, -6.911683, -6.890206, -6.571945, + -5.766199, -5.629292, -5.500316, -5.457587, -5.137841, -4.946566, + -4.747363, -4.453594, -4.333026, -4.211693, + 1.063227, 1.156564, 1.210127, 1.265683, 1.325263, 1.37135, 1.379399, + 1.395321, 1.408047, 1.413615, + 0.8057297, 0.823011, 0.8517545, 0.8839636, 0.8734952, 0.8716547, 0.8792083, + 0.8959213, 0.933274, 0.9308779, + 0.3776965, 0.3511574, 0.3064211, 0.2771505, 0.2723292, 0.26952, 0.2818037, + 0.2319987, 0.165376, 0.1122594, + -0.2686788, -0.3904975, -0.4879307, -0.5713966, -0.6487127, -0.7579689, + -0.879656, -1.019151, -1.160378, -1.290107, + -1.44353, -1.544772, -1.67499, -1.812897, -1.98716, -2.274205, -2.556359, + -2.771371, -2.97445, -3.201544, + -3.11445, -3.268162, -3.425929, -3.644483, -3.970347, -4.194717, -4.426955, + -4.785159, -4.986393, -5.121875, + -4.842557, -4.942987, -5.010421, -5.153152, -5.254729, -5.326348, + -5.648958, -6.018217, -6.451276, -6.772595, + -6.110239, -6.028393, -5.95482, -5.914704, -5.883089, -5.694447, -5.721324, + -5.85588, -6.038979, -6.222065, + -6.000411, -6.046391, -5.883338, -5.710005, -5.577016, -5.360312, + -5.450391, -5.419651, -5.09429, -4.927251, + -4.122381, -4.0073, -3.920688, -3.890532, -3.619308, -3.348911, -3.164834, + -3.032901, -2.999816, -2.981941, + 1.021295, 1.058732, 1.055263, 1.055828, 1.06735, 1.085502, 1.072789, + 1.066372, 1.055719, 1.03737, + 0.5644462, 0.5258871, 0.505933, 0.5047766, 0.4665931, 0.4326979, 0.4054725, + 0.3554713, 0.3202047, 0.2826239, + -0.09909091, -0.1695052, -0.2423104, -0.308107, -0.3618009, -0.4362217, + -0.5038798, -0.5690737, -0.6322247, -0.6912587, + -0.8452172, -0.9907208, -1.12432, -1.275977, -1.407145, -1.514558, + -1.611202, -1.70489, -1.787341, -1.840466, + -1.660944, -1.836267, -2.043581, -2.208486, -2.365314, -2.544751, + -2.701359, -2.791034, -2.877407, -2.950873, + -2.49663, -2.691726, -2.876794, -3.089699, -3.315934, -3.422023, -3.487426, + -3.497974, -3.590583, -3.699841, + -3.409052, -3.454748, -3.579855, -3.723231, -3.777681, -3.849489, + -3.761075, -3.758978, -3.830392, -3.977781, + -4.332948, -4.30478, -4.268668, -4.142052, -4.038466, -3.906201, -3.853595, + -3.97838, -4.061214, -3.96847, + -4.037989, -3.923016, -3.725387, -3.504999, -3.335768, -3.048127, + -2.881483, -2.777315, -2.57696, -2.525698, + -2.675588, -2.521989, -2.357147, -2.217182, -1.92482, -1.59737, -1.311503, + -1.052147, -0.8554193, -0.7043628, + 0.868393, 0.8096983, 0.7342755, 0.6706711, 0.6231919, 0.5861326, 0.5235301, + 0.4832453, 0.4458269, 0.403123, + 0.3249699, 0.203376, 0.09710471, 0.005840228, -0.09959703, -0.186581, + -0.2659973, -0.3638777, -0.4377598, -0.5044919, + -0.4682975, -0.6448929, -0.7865676, -0.9014125, -1.010695, -1.141248, + -1.26495, -1.354475, -1.432743, -1.508723, + -1.302762, -1.487646, -1.667504, -1.870815, -2.038251, -2.165048, + -2.268736, -2.35599, -2.422225, -2.462221, + -1.897165, -2.089815, -2.293899, -2.467101, -2.626223, -2.795213, + -2.938092, -3.009527, -3.077491, -3.088423, + -2.338803, -2.44815, -2.576407, -2.714023, -2.881914, -2.990129, -3.088588, + -3.162073, -3.200975, -3.280721, + -2.629382, -2.617197, -2.644625, -2.714817, -2.728057, -2.764721, + -2.755495, -2.804715, -2.900975, -3.060241, + -2.806314, -2.699975, -2.604603, -2.467385, -2.316504, -2.181494, + -2.104434, -2.068235, -2.089983, -2.092961, + -2.350076, -2.191863, -2.015235, -1.808842, -1.591449, -1.298141, + -1.055869, -0.8611979, -0.6329716, -0.4672858, + -1.417739, -1.253469, -1.085745, -0.9032833, -0.6274295, -0.3408952, + -0.04521877, 0.2827266, 0.5437686, 0.7892594, + 0.08603396, -0.01244956, -0.1302382, -0.230692, -0.3224648, -0.4244072, + -0.5279509, -0.5862728, -0.6336403, -0.6853408, + -0.5083337, -0.6752201, -0.8401579, -1.021567, -1.198536, -1.330774, + -1.454516, -1.586487, -1.691136, -1.77504, + -1.209975, -1.465892, -1.672402, -1.836556, -2.003343, -2.175733, + -2.340954, -2.462788, -2.570125, -2.671328, + -1.835453, -2.040931, -2.250755, -2.488194, -2.693811, -2.848905, + -2.973454, -3.076011, -3.155657, -3.221975, + -2.087997, -2.262499, -2.42809, -2.583258, -2.728339, -2.874663, -3.012026, + -3.105144, -3.219588, -3.292633, + -2.219754, -2.239856, -2.291842, -2.338402, -2.416109, -2.477032, + -2.551712, -2.625169, -2.696625, -2.812562, + -2.214184, -2.120897, -2.02604, -1.978979, -1.914884, -1.861667, -1.797479, + -1.808812, -1.854385, -1.959785, + -2.145867, -1.937978, -1.761547, -1.585332, -1.406883, -1.215052, + -1.064301, -0.9400889, -0.883372, -0.8476374, + -1.6123, -1.449645, -1.277987, -1.08922, -0.8639328, -0.5785105, + -0.3019016, -0.06757384, 0.1769251, 0.3469639, + -0.4829444, -0.3573219, -0.2495783, -0.1331687, 0.03723535, 0.2394148, + 0.4937723, 0.8273596, 1.081453, 1.332125, + -0.8436456, -0.9486177, -1.059377, -1.153291, -1.241334, -1.344937, + -1.442711, -1.494026, -1.531425, -1.569082, + -1.363601, -1.517111, -1.673566, -1.853703, -2.024385, -2.153884, + -2.277348, -2.399244, -2.49864, -2.581057, + -1.900008, -2.121459, -2.298829, -2.446108, -2.600936, -2.75574, -2.91209, + -3.041326, -3.159974, -3.291958, + -2.332246, -2.485257, -2.649413, -2.831298, -2.996419, -3.138222, + -3.260611, -3.390049, -3.512205, -3.606771, + -2.484051, -2.582164, -2.656566, -2.747351, -2.843552, -2.953647, + -3.080211, -3.176885, -3.30392, -3.404524, + -2.524805, -2.482363, -2.478343, -2.466771, -2.482541, -2.493074, + -2.524661, -2.568022, -2.622594, -2.706606, + -2.361249, -2.253306, -2.134065, -2.054415, -1.964403, -1.889408, + -1.791761, -1.750293, -1.718029, -1.732389, + -2.014955, -1.839144, -1.68769, -1.535044, -1.384408, -1.211162, -1.050033, + -0.8907735, -0.7928692, -0.7119153, + -1.236629, -1.093662, -0.9508283, -0.8085872, -0.6442373, -0.4315634, + -0.1945382, -0.002699569, 0.1975364, 0.3391424, + -0.113028, 0.03701114, 0.1509672, 0.244096, 0.3440107, 0.4850455, 0.659207, + 0.8812482, 1.058411, 1.222257, + -1.854996, -1.890166, -1.915619, -1.932104, -1.946811, -1.971547, + -1.988159, -1.97958, -1.964028, -1.948233, + -2.224256, -2.271724, -2.324106, -2.393641, -2.449779, -2.481474, + -2.509733, -2.527462, -2.532758, -2.527077, + -2.520393, -2.60658, -2.663467, -2.701849, -2.744738, -2.780796, -2.817967, + -2.841355, -2.859165, -2.87995, + -2.696981, -2.736231, -2.781672, -2.833747, -2.878667, -2.9107, -2.931204, + -2.952544, -2.963497, -2.964039, + -2.669573, -2.676523, -2.663651, -2.663748, -2.672803, -2.703568, -2.73891, + -2.744817, -2.765156, -2.767472, + -2.476536, -2.429364, -2.405869, -2.377028, -2.36033, -2.332005, -2.306961, + -2.277795, -2.2701, -2.287573, + -2.063781, -1.986485, -1.90999, -1.85632, -1.792651, -1.720523, -1.636689, + -1.607912, -1.594598, -1.61015, + -1.519658, -1.429246, -1.343379, -1.238522, -1.116604, -1.015946, + -0.9358807, -0.8798366, -0.8500427, -0.8273263, + -0.7674065, -0.6452081, -0.5515807, -0.4693587, -0.3846785, -0.3116637, + -0.2242767, -0.1593899, -0.1116388, -0.06896764, + -0.01183324, 0.1091698, 0.2188423, 0.312474, 0.3854575, 0.4438081, + 0.5140151, 0.5824342, 0.6187019, 0.6379458, + -2.684902, -2.610719, -2.535948, -2.460751, -2.387049, -2.321111, + -2.254766, -2.182338, -2.107151, -2.031494, + -2.820069, -2.740268, -2.659949, -2.584899, -2.516798, -2.444206, + -2.371544, -2.295798, -2.221301, -2.142587, + -2.920424, -2.834318, -2.745224, -2.660668, -2.577662, -2.494831, -2.4213, + -2.345643, -2.268257, -2.192751, + -2.919983, -2.838565, -2.757891, -2.677184, -2.599998, -2.519447, + -2.432564, -2.343707, -2.255901, -2.170567, + -2.795936, -2.718457, -2.634572, -2.554108, -2.473378, -2.391261, + -2.312509, -2.233216, -2.158018, -2.077407, + -2.562569, -2.481418, -2.403707, -2.320077, -2.236581, -2.159199, + -2.083839, -2.011575, -1.949767, -1.892647, + -2.250703, -2.162687, -2.078802, -2.00374, -1.924842, -1.843397, -1.762613, + -1.697863, -1.633513, -1.583002, + -1.825546, -1.773501, -1.720045, -1.646263, -1.569409, -1.491826, + -1.415384, -1.344221, -1.289576, -1.235646, + -1.16891, -1.113447, -1.082653, -1.06613, -1.038724, -1.014261, -0.971841, + -0.9325086, -0.8841423, -0.8362257, + -0.4466377, -0.3950162, -0.3640113, -0.3467484, -0.3497896, -0.3435763, + -0.3285452, -0.3093777, -0.3015329, -0.2992152, + -3.47265, -3.349887, -3.225127, -3.099491, -2.975098, -2.858005, -2.742823, + -2.620348, -2.497191, -2.375009, + -3.669024, -3.520918, -3.372771, -3.233594, -3.098513, -2.952549, + -2.805939, -2.665119, -2.521485, -2.376583, + -3.875113, -3.713779, -3.561024, -3.408706, -3.2558, -3.100774, -2.95319, + -2.807744, -2.659667, -2.510706, + -3.950386, -3.795719, -3.641412, -3.475792, -3.323338, -3.181724, + -3.039761, -2.904638, -2.775108, -2.650002, + -3.947389, -3.783504, -3.606038, -3.452417, -3.30665, -3.171729, -3.049297, + -2.93359, -2.822966, -2.726708, + -3.906445, -3.746139, -3.594896, -3.442577, -3.297046, -3.1648, -3.036906, + -2.916452, -2.812173, -2.712864, + -3.83857, -3.71836, -3.602963, -3.481781, -3.346611, -3.205612, -3.061509, + -2.923938, -2.794568, -2.691266, + -3.559861, -3.530004, -3.478284, -3.396861, -3.302307, -3.188716, + -3.069149, -2.94336, -2.817003, -2.663864, + -2.915129, -2.919575, -2.929287, -2.93514, -2.908199, -2.866826, -2.797559, + -2.709057, -2.578434, -2.428774, + -2.010599, -2.042224, -2.077676, -2.104624, -2.122853, -2.120454, + -2.097497, -2.047853, -1.992431, -1.92157, + -4.22112, -4.07559, -3.922843, -3.768222, -3.614719, -3.473573, -3.340809, + -3.200793, -3.061464, -2.926893, + -4.502892, -4.344226, -4.183433, -4.026276, -3.878217, -3.729609, -3.58069, + -3.431968, -3.287462, -3.145941, + -4.631217, -4.473382, -4.315218, -4.164486, -4.013389, -3.85793, -3.710944, + -3.570176, -3.429156, -3.28708, + -4.626989, -4.476557, -4.326241, -4.174338, -4.032887, -3.895635, + -3.756263, -3.619544, -3.485802, -3.356973, + -4.556738, -4.395508, -4.235921, -4.096162, -3.958964, -3.832275, + -3.715655, -3.600988, -3.489401, -3.387974, + -4.468916, -4.302645, -4.15163, -4.001896, -3.865624, -3.743882, -3.626797, + -3.521506, -3.429111, -3.338457, + -4.44977, -4.292527, -4.144361, -4.001679, -3.853956, -3.707088, -3.567008, + -3.438192, -3.317925, -3.220261, + -4.439157, -4.329401, -4.202333, -4.056956, -3.905393, -3.736642, + -3.568599, -3.403389, -3.260043, -3.114572, + -4.203069, -4.12499, -4.031964, -3.926179, -3.793952, -3.63649, -3.470912, + -3.312045, -3.139288, -2.968029, + -3.549062, -3.518938, -3.466148, -3.407319, -3.308811, -3.204609, + -3.090285, -2.963688, -2.832902, -2.702063, + -4.634931, -4.510322, -4.387384, -4.262967, -4.13814, -4.019274, -3.907151, + -3.793168, -3.677498, -3.562162, + -4.812496, -4.685526, -4.557202, -4.432652, -4.315062, -4.196854, + -4.077685, -3.961596, -3.849445, -3.737238, + -4.926235, -4.801914, -4.682349, -4.564368, -4.445381, -4.328474, + -4.217326, -4.105275, -3.991226, -3.87654, + -4.974308, -4.86126, -4.746056, -4.629657, -4.521437, -4.412365, -4.300677, + -4.186651, -4.075935, -3.968378, + -4.938999, -4.831478, -4.722439, -4.619403, -4.516479, -4.414182, + -4.313823, -4.215149, -4.114635, -4.01406, + -4.802555, -4.709178, -4.621257, -4.533046, -4.446228, -4.362994, + -4.278905, -4.198927, -4.114691, -4.029364, + -4.646967, -4.549815, -4.467683, -4.393793, -4.318109, -4.249639, + -4.184724, -4.118573, -4.056467, -3.996501, + -4.553621, -4.461698, -4.370646, -4.279413, -4.197869, -4.108099, + -4.025449, -3.951957, -3.882595, -3.806807, + -4.450593, -4.350024, -4.254059, -4.160119, -4.047391, -3.930037, + -3.814608, -3.705483, -3.598191, -3.498837, + -4.176404, -4.088185, -3.99167, -3.888712, -3.763858, -3.636919, -3.503574, + -3.371032, -3.253147, -3.15005, + -5.305916, -5.228198, -5.149081, -5.066242, -4.979733, -4.891483, + -4.805131, -4.715399, -4.625538, -4.536609, + -5.409827, -5.338651, -5.264568, -5.190886, -5.118539, -5.040178, + -4.958767, -4.879787, -4.798156, -4.714354, + -5.436405, -5.377537, -5.321352, -5.260819, -5.19785, -5.135309, -5.069129, + -4.99902, -4.925947, -4.848398, + -5.425586, -5.371675, -5.315626, -5.258965, -5.205391, -5.147702, + -5.090356, -5.029489, -4.966897, -4.903516, + -5.421552, -5.364684, -5.306411, -5.247279, -5.189979, -5.13738, -5.082496, + -5.023454, -4.961823, -4.898187, + -5.399646, -5.353481, -5.305019, -5.255711, -5.203397, -5.146911, + -5.086962, -5.023842, -4.958927, -4.891607, + -5.324206, -5.285987, -5.253324, -5.216431, -5.17214, -5.119621, -5.066675, + -5.011673, -4.952978, -4.889044, + -5.201594, -5.185238, -5.161349, -5.129875, -5.091774, -5.047886, + -5.002157, -4.954802, -4.903096, -4.845263, + -5.027672, -5.009829, -4.996872, -4.982873, -4.957479, -4.928816, + -4.894786, -4.855186, -4.809749, -4.755376, + -4.800905, -4.790343, -4.778637, -4.76788, -4.754467, -4.730603, -4.70276, + -4.672376, -4.637187, -4.600635, + -5.877733, -5.806942, -5.737911, -5.665781, -5.589697, -5.508978, + -5.428902, -5.346344, -5.262498, -5.177478, + -5.999748, -5.93775, -5.871769, -5.80268, -5.736236, -5.664495, -5.589656, + -5.515277, -5.436458, -5.355067, + -6.105768, -6.050148, -5.99489, -5.934047, -5.869964, -5.807178, -5.740132, + -5.669778, -5.59723, -5.518231, + -6.204219, -6.156556, -6.104576, -6.049919, -5.996562, -5.93629, -5.873829, + -5.805651, -5.734739, -5.662208, + -6.283307, -6.238613, -6.188774, -6.136417, -6.083857, -6.031115, + -5.974864, -5.914575, -5.849953, -5.782243, + -6.327734, -6.29085, -6.250577, -6.207967, -6.160203, -6.109562, -6.055408, + -5.996362, -5.935081, -5.871264, + -6.328987, -6.301887, -6.276907, -6.244695, -6.205217, -6.156336, + -6.106204, -6.054013, -5.999475, -5.941885, + -6.286625, -6.282731, -6.267518, -6.242774, -6.20835, -6.168219, -6.125497, + -6.080946, -6.031807, -5.975547, + -6.193247, -6.187342, -6.183197, -6.175198, -6.155711, -6.131026, -6.09807, + -6.059319, -6.013174, -5.959172, + -6.058994, -6.055746, -6.05086, -6.046446, -6.038727, -6.018096, -5.98981, + -5.954857, -5.918507, -5.881165, + -6.561323, -6.499266, -6.438377, -6.375708, -6.310773, -6.244139, + -6.180099, -6.115896, -6.051152, -5.986147, + -6.845045, -6.786884, -6.725439, -6.661109, -6.601454, -6.540991, + -6.479097, -6.417562, -6.355603, -6.293946, + -7.102453, -7.045627, -6.989235, -6.932842, -6.874574, -6.816972, + -6.758873, -6.701811, -6.644189, -6.583754, + -7.351357, -7.301179, -7.248499, -7.19398, -7.142448, -7.089523, -7.035904, + -6.981055, -6.926404, -6.871471, + -7.598204, -7.549111, -7.497645, -7.447221, -7.397934, -7.350796, + -7.304009, -7.256029, -7.20568, -7.155462, + -7.82113, -7.78062, -7.739799, -7.69751, -7.652907, -7.609156, -7.56383, + -7.517503, -7.472572, -7.428266, + -8.020307, -7.985536, -7.955499, -7.923059, -7.885509, -7.841635, + -7.801394, -7.76416, -7.727195, -7.691062, + -8.203031, -8.185472, -8.161825, -8.130711, -8.093474, -8.056222, + -8.020072, -7.988377, -7.958443, -7.925798, + -8.343966, -8.324842, -8.308795, -8.291328, -8.267859, -8.240988, -8.21365, + -8.186219, -8.156057, -8.126547, + -8.44836, -8.431485, -8.415438, -8.404657, -8.391961, -8.373449, -8.350844, + -8.326425, -8.308137, -8.293455, + -7.418903, -7.380289, -7.343199, -7.30528, -7.266068, -7.225653, -7.186882, + -7.148771, -7.1104, -7.071782, + -7.886793, -7.851552, -7.814201, -7.774897, -7.739104, -7.703277, + -7.666594, -7.630272, -7.594023, -7.558314, + -8.341415, -8.308264, -8.274553, -8.240778, -8.205718, -8.171395, + -8.137189, -8.103981, -8.07057, -8.035443, + -8.796901, -8.767877, -8.73701, -8.704835, -8.674766, -8.644069, -8.61279, + -8.580782, -8.54905, -8.517601, + -9.253543, -9.225341, -9.195226, -9.165483, -9.137247, -9.111868, + -9.086359, -9.059315, -9.030994, -9.003092, + -9.699064, -9.677286, -9.654745, -9.63162, -9.607246, -9.582701, -9.557117, + -9.531214, -9.505648, -9.481256, + -10.12736, -10.10717, -10.09253, -10.07685, -10.05586, -10.03019, + -10.00788, -9.988334, -9.96856, -9.949014, + -10.53594, -10.52901, -10.51833, -10.50091, -10.47864, -10.45729, + -10.43758, -10.42123, -10.40631, -10.38915, + -10.90329, -10.89263, -10.88709, -10.88161, -10.87157, -10.86071, + -10.84752, -10.83391, -10.81815, -10.80181, + -11.248, -11.23781, -11.23062, -11.22926, -11.23034, -11.22393, -11.21365, + -11.20186, -11.19413, -11.18977, + -13.83239, -13.82074, -13.808, -13.79507, -13.78222, -13.76985, -13.75674, + -13.74387, -13.73124, -13.71895, + -14.99369, -14.98229, -14.97097, -14.96008, -14.94822, -14.9366, -14.92516, + -14.9145, -14.90532, -14.89437, + -16.12242, -16.11533, -16.10389, -16.09248, -16.08173, -16.06978, + -16.06084, -16.05067, -16.03975, -16.03102, + -17.2117, -17.20174, -17.1923, -17.18274, -17.17234, -17.16337, -17.15279, + -17.14514, -17.13551, -17.12395, + -18.23167, -18.22563, -18.21396, -18.20369, -18.19536, -18.19025, -18.1865, + -18.17599, -18.1694, -18.16037, + -19.16084, -19.15458, -19.15068, -19.14543, -19.14352, -19.13608, + -19.12984, -19.12499, -19.11406, -19.10741, + -19.96635, -19.96284, -19.96158, -19.96541, -19.9607, -19.95777, -19.95134, + -19.94964, -19.94552, -19.9427, + -20.64782, -20.64862, -20.65172, -20.64795, -20.64328, -20.63709, -20.6386, + -20.63893, -20.64199, -20.63851, + -21.1739, -21.17439, -21.17778, -21.18302, -21.18665, -21.19431, -21.1958, + -21.1981, -21.19873, -21.19386, + -21.59861, -21.59514, -21.59659, -21.60578, -21.61945, -21.62407, + -21.62483, -21.62446, -21.6267, -21.63452 ; +} diff --git a/fremorizer/tests/test_files/reduced_ascii_files/reduced_ocean_monthly_1x1deg.199301-199302.sos.cdl b/fremorizer/tests/test_files/reduced_ascii_files/reduced_ocean_monthly_1x1deg.199301-199302.sos.cdl new file mode 100644 index 0000000..a1a8a64 --- /dev/null +++ b/fremorizer/tests/test_files/reduced_ascii_files/reduced_ocean_monthly_1x1deg.199301-199302.sos.cdl @@ -0,0 +1,84 @@ +netcdf reduced_ocean_monthly_1x1deg.199301-199302.sos { +dimensions: + lat = 2 ; + bnds = 2 ; + lon = 2 ; + time = UNLIMITED ; // (2 currently) +variables: + double lat(lat) ; + lat:long_name = "latitude" ; + lat:units = "degrees_N" ; + lat:axis = "Y" ; + lat:bounds = "lat_bnds" ; + double lat_bnds(lat, bnds) ; + lat_bnds:long_name = "latitude bounds" ; + lat_bnds:units = "degrees_N" ; + lat_bnds:axis = "Y" ; + double lon(lon) ; + lon:long_name = "longitude" ; + lon:units = "degrees_E" ; + lon:axis = "X" ; + lon:bounds = "lon_bnds" ; + double lon_bnds(lon, bnds) ; + lon_bnds:long_name = "longitude bounds" ; + lon_bnds:units = "degrees_E" ; + lon_bnds:axis = "X" ; + float sos(time, lat, lon) ; + sos:_FillValue = 1.e+20f ; + sos:missing_value = 1.e+20f ; + sos:units = "psu" ; + sos:long_name = "Sea Surface Salinity" ; + sos:cell_methods = "area:mean yh:mean xh:mean time: mean" ; + sos:cell_measures = "area: areacello" ; + sos:standard_name = "sea_surface_salinity" ; + sos:interp_method = "conserve_order1" ; + double time(time) ; + time:units = "days since 1958-01-01 00:00:00" ; + time:long_name = "time" ; + time:axis = "T" ; + time:calendar_type = "JULIAN" ; + time:calendar = "julian" ; + time:bounds = "time_bnds" ; + double time_bnds(time, bnds) ; + time_bnds:units = "days since 1958-01-01 00:00:00" ; + time_bnds:long_name = "time axis boundaries" ; + +// global attributes: + :title = "om5_b05_noHiLatHenyey_55" ; + :associated_files = "areacello: 19930101.ocean_static.nc" ; + :grid_type = "regular" ; + :grid_tile = "N/A" ; + :code_release_version = "2024.02" ; + :git_hash = "b86d27037f755a82c586e55073dd575245c144b1" ; + :creationtime = "Mon Jun 17 18:57:22 2024" ; + :hostname = "pp337" ; + :history = "Wed Nov 6 19:17:52 2024: ncks -d lat,0,1 -d lon,0,1 -d time,0,1 ocean_monthly_1x1deg.199301-199712.sos.nc -o reduced_ocean_monthly_1x1deg.199301-199302.sos.nc\n", + "fregrid --standard_dimension --input_mosaic ocean_mosaic.nc --input_file all --interp_method conserve_order1 --remap_file .fregrid_remap_file_360_by_180.nc --nlon 360 --nlat 180 --scalar_field (**please see the field list in this file**) --output_file out.nc" ; + :external_variables = "areacello" ; + :NCO = "netCDF Operators version 5.1.5 (Homepage = http://nco.sf.net, Code = http://github.com/nco/nco)" ; +data: + + lat = -89.5, -88.5 ; + + lat_bnds = + -90, -89, + -89, -88 ; + + lon = 0.5, 1.5 ; + + lon_bnds = + 0, 1, + 1, 2 ; + + sos = + 35.5, 36.1, + 35.2, 33.8, + 32.6, 34.2, + 33.9, 35.7 ; + + time = 12799.5, 12829 ; + + time_bnds = + 12784, 12815, + 12815, 12843 ; +} diff --git a/fremorizer/tests/test_files/varlist b/fremorizer/tests/test_files/varlist new file mode 100644 index 0000000..6ab9e86 --- /dev/null +++ b/fremorizer/tests/test_files/varlist @@ -0,0 +1,3 @@ +{ + "sos": "sos" +} diff --git a/fremorizer/tests/test_files/varlist_local_target_vars_differ b/fremorizer/tests/test_files/varlist_local_target_vars_differ new file mode 100644 index 0000000..80fa153 --- /dev/null +++ b/fremorizer/tests/test_files/varlist_local_target_vars_differ @@ -0,0 +1,4 @@ +{ + "sosV2": "sos", + "sos": "sosTYPO" +} diff --git a/meta.yaml b/meta.yaml new file mode 100644 index 0000000..24bffcf --- /dev/null +++ b/meta.yaml @@ -0,0 +1,35 @@ +package: + name: fremorizer + version: 2026.01.alpha1 + +source: + path: . + +build: + script: "{{ PYTHON }} -m pip install . -vv" + number: 0 + noarch: python +requirements: + host: + - python >=3.11 + - pip + run: + - python >=3.11 + - cftime + - click + - cmor + - netcdf4 + - numpy + - pyyaml +test: + imports: + - fremorizer + - fremorizer.cmor_mixer + - fremorizer.cmor_helpers + - fremorizer.cmor_finder + - fremorizer.cmor_config + - fremorizer.cmor_yamler +about: + home: https://github.com/ilaflott/fremorizer + license: Apache-2.0 + summary: Model output rewriter (CMORizer) for FRE/FMS based models diff --git a/pylintrc b/pylintrc new file mode 100644 index 0000000..ac870b9 --- /dev/null +++ b/pylintrc @@ -0,0 +1,646 @@ +[MAIN] + +# Analyze import fallback blocks. This can be used to support both Python 2 and +# 3 compatible code, which means that the block might have code that exists +# only in one or another interpreter, leading to false positives when analyzed. +analyse-fallback-blocks=no # cspell:disable-line + +# Clear in-memory caches upon conclusion of linting. Useful if running pylint +# in a server-like mode. +clear-cache-post-run=no + +# Load and enable all available extensions. Use --list-extensions to see a list +# all available extensions. +#enable-all-extensions= + +# In error mode, messages with a category besides ERROR or FATAL are +# suppressed, and no reports are done by default. Error mode is compatible with +# disabling specific errors. +#errors-only= + +# Always return a 0 (non-error) status code, even if lint errors are found. +# This is primarily useful in continuous integration scripts. +#exit-zero= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. +extension-pkg-allow-list= + +# A comma-separated list of package or module names from where C extensions may +# be loaded. Extensions are loading into the active Python interpreter and may +# run arbitrary code. (This is an alternative name to extension-pkg-allow-list +# for backward compatibility.) +extension-pkg-whitelist= + +# Return non-zero exit code if any of these messages/categories are detected, +# even if score is above --fail-under value. Syntax same as enable. Messages +# specified are enabled, while categories only check already-enabled messages. +fail-on= + +# Specify a score threshold under which the program will exit with error. +fail-under=9.0 + +# Interpret the stdin as a python script, whose filename needs to be passed as +# the module_or_package argument. +#from-stdin= + +# Files or directories to be skipped. They should be base names, not paths. +ignore=CVS + +# Add files or directories matching the regular expressions patterns to the +# ignore-list. The regex matches against paths and can be in Posix or Windows +# format. Because '\\' represents the directory delimiter on Windows systems, +# it can't be used as an escape character. +ignore-paths= + +# Files or directories matching the regular expression patterns are skipped. +# The regex matches against base names, not paths. The default value ignores +# Emacs file locks +ignore-patterns=^\.# + +# List of module names for which member attributes should not be checked and +# will not be imported (useful for modules/projects where namespaces are +# manipulated during runtime and thus existing member attributes cannot be +# deduced by static analysis). It supports qualified module names, as well as +# Unix pattern matching. +ignored-modules=netCDF4,cmor + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Use multiple processes to speed up Pylint. Specifying 0 will auto-detect the +# number of processors available to use, and will cap the count on Windows to +# avoid hangs. +jobs=1 + +# Control the amount of potential inferred values when inferring a single +# object. This can help the performance when dealing with large functions or +# complex, nested conditions. +limit-inference-results=100 + +# List of plugins (as comma separated values of python module names) to load, +# usually to register additional checkers. +load-plugins= + +# Pickle collected data for later comparisons. +persistent=yes + +# Resolve imports to .pyi stubs if available. May reduce no-member messages and +# increase not-an-iterable messages. +prefer-stubs=no + +# Minimum Python version to use for version dependent checks. Will default to +# the version used to run pylint. +py-version=3.11 + +# Discover python modules and packages in the file system subtree. +recursive=no + +# Add paths to the list of the source roots. Supports globbing patterns. The +# source root is an absolute path or a path relative to the current working +# directory used to determine a package namespace for modules located under the +# source root. +source-roots= + +## When enabled, pylint would attempt to guess common misconfiguration and emit +## user-friendly hints instead of false-positive error messages. +#suggestion-mode=yes + +# Allow loading of arbitrary C extensions. Extensions are imported into the +# active Python interpreter and may run arbitrary code. +unsafe-load-any-extension=no + +# In verbose mode, extra non-checker-related info will be displayed. +#verbose= + + +[BASIC] + +# Naming style matching correct argument names. +argument-naming-style=snake_case + +# Regular expression matching correct argument names. Overrides argument- +# naming-style. If left empty, argument names will be checked with the set +# naming style. +#argument-rgx= + +# Naming style matching correct attribute names. +attr-naming-style=snake_case + +# Regular expression matching correct attribute names. Overrides attr-naming- +# style. If left empty, attribute names will be checked with the set naming +# style. +#attr-rgx= + +# Bad variable names which should always be refused, separated by a comma. +bad-names=foo, + bar, + baz, + toto, + tutu, + tata + +# Bad variable names regexes, separated by a comma. If names match any regex, +# they will always be refused +bad-names-rgxs= + +# Naming style matching correct class attribute names. +class-attribute-naming-style=any + +# Regular expression matching correct class attribute names. Overrides class- +# attribute-naming-style. If left empty, class attribute names will be checked +# with the set naming style. +#class-attribute-rgx= + +# Naming style matching correct class constant names. +class-const-naming-style=UPPER_CASE + +# Regular expression matching correct class constant names. Overrides class- +# const-naming-style. If left empty, class constant names will be checked with +# the set naming style. +#class-const-rgx= + +# Naming style matching correct class names. +class-naming-style=PascalCase + +# Regular expression matching correct class names. Overrides class-naming- +# style. If left empty, class names will be checked with the set naming style. +#class-rgx= + +# Naming style matching correct constant names. +const-naming-style=UPPER_CASE + +# Regular expression matching correct constant names. Overrides const-naming- +# style. If left empty, constant names will be checked with the set naming +# style. +#const-rgx= + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + +# Naming style matching correct function names. +function-naming-style=snake_case + +# Regular expression matching correct function names. Overrides function- +# naming-style. If left empty, function names will be checked with the set +# naming style. +#function-rgx= + +# Good variable names which should always be accepted, separated by a comma. +good-names=i, + j, + k, + ex, + Run, + _ + +# Good variable names regexes, separated by a comma. If names match any regex, +# they will always be accepted +good-names-rgxs= + +# Include a hint for the correct naming format with invalid-name. +include-naming-hint=no + +# Naming style matching correct inline iteration names. +inlinevar-naming-style=any + +# Regular expression matching correct inline iteration names. Overrides +# inlinevar-naming-style. If left empty, inline iteration names will be checked +# with the set naming style. +#inlinevar-rgx= + +# Naming style matching correct method names. +method-naming-style=snake_case + +# Regular expression matching correct method names. Overrides method-naming- +# style. If left empty, method names will be checked with the set naming style. +#method-rgx= + +# Naming style matching correct module names. +module-naming-style=snake_case + +# Regular expression matching correct module names. Overrides module-naming- +# style. If left empty, module names will be checked with the set naming style. +#module-rgx= + +# Colon-delimited sets of names that determine each other's naming style when +# the name regexes allow several styles. +name-group= + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=^_ + +# List of decorators that produce properties, such as abc.abstractproperty. Add +# to this list to register other decorators that produce valid properties. +# These decorators are taken in consideration only for invalid-name. +property-classes=abc.abstractproperty + +# Regular expression matching correct type alias names. If left empty, type +# alias names will be checked with the set naming style. +#typealias-rgx= + +# Regular expression matching correct type variable names. If left empty, type +# variable names will be checked with the set naming style. +#typevar-rgx= + +# Naming style matching correct variable names. +variable-naming-style=snake_case + +# Regular expression matching correct variable names. Overrides variable- +# naming-style. If left empty, variable names will be checked with the set +# naming style. +#variable-rgx= + + +[CLASSES] + +# Warn about protected attribute access inside special methods +check-protected-access-in-special-methods=no + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__, + __new__, + setUp, + asyncSetUp, + __post_init__ + +# List of member names, which should be excluded from the protected access +# warning. +exclude-protected=_asdict,_fields,_replace,_source,_make,os._exit + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# List of regular expressions of class ancestor names to ignore when counting +# public methods (see R0903) +exclude-too-few-public-methods= + +# List of qualified class names to ignore when counting class parents (see +# R0901) +ignored-parents= + +# Maximum number of arguments for function / method. +max-args=6 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Maximum number of boolean expressions in an if statement (see R0916). +max-bool-expr=5 + +# Maximum number of branch for function / method body. +max-branches=12 + +# Maximum number of locals for function / method body. +max-locals=15 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of positional arguments for function / method. +max-positional-arguments=6 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + +# Maximum number of return / yield for function / method body. +max-returns=6 + +# Maximum number of statements in function / method body. +max-statements=50 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when caught. +overgeneral-exceptions=builtins.BaseException,builtins.Exception + + +[FORMAT] + +# Expected format of line ending, e.g. empty (any line ending), LF or CRLF. +expected-line-ending-format= + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Number of spaces of indent required inside a hanging or continued line. +indent-after-paren=4 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + +# Maximum number of characters on a single line. +max-line-length=120 + +# Maximum number of lines in a module. +max-module-lines=1000 + +# Allow the body of a class to be on the same line as the declaration if body +# contains single statement. +single-line-class-stmt=no + +# Allow the body of an if to be on the same line as the test if there is no +# else. +single-line-if-stmt=no + + +[IMPORTS] + +# List of modules that can be imported at any level, not just the top level +# one. +allow-any-import-level= + +# Allow explicit reexports by alias from a package __init__. +allow-reexport-from-package=no + +# Allow wildcard imports from modules that define __all__. +allow-wildcard-with-all=no + +# Deprecated modules which should not be used, separated by a comma. +deprecated-modules= + +# Output a graph (.gv or any supported image format) of external dependencies +# to the given file (report RP0402 must not be disabled). +ext-import-graph= + +# Output a graph (.gv or any supported image format) of all (i.e. internal and +# external) dependencies to the given file (report RP0402 must not be +# disabled). +import-graph= + +# Output a graph (.gv or any supported image format) of internal dependencies +# to the given file (report RP0402 must not be disabled). +int-import-graph= + +# Force import order to recognize a module as part of the standard +# compatibility libraries. +known-standard-library= + +# Force import order to recognize a module as part of a third party library. +known-third-party=enchant + +# Couples of modules and preferred modules, separated by a comma. +preferred-modules= + + +[LOGGING] + +# The type of string formatting that logging methods do. `old` means using % +# formatting, `new` is for `{}` formatting. +logging-format-style=old + +# Logging modules to check that the string format arguments are in logging +# function parameter format. +logging-modules=logging + + +[MESSAGES CONTROL] + +# Only show warnings with the listed confidence levels. Leave empty to show +# all. Valid levels: HIGH, CONTROL_FLOW, INFERENCE, INFERENCE_FAILURE, +# UNDEFINED. +confidence=HIGH, + CONTROL_FLOW, + INFERENCE, + INFERENCE_FAILURE, + UNDEFINED + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once). You can also use "--disable=all" to +# disable everything first and then re-enable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use "--disable=all --enable=classes +# --disable=W". +disable=raw-checker-failed, + bad-inline-option, + locally-disabled, + file-ignored, + suppressed-message, + useless-suppression, + deprecated-pragma, + use-implicit-booleaness-not-comparison-to-string, + use-implicit-booleaness-not-comparison-to-zero, + use-symbolic-message-instead + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time (only on the command line, not in the configuration file where +# it should appear only once). See also the "--disable" option for examples. +enable= + + +[METHOD_ARGS] + +# List of qualified names (i.e., library.method) which require a timeout +# parameter e.g. 'requests.api.get,requests.api.post' +timeout-methods=requests.api.delete,requests.api.get,requests.api.head,requests.api.options,requests.api.patch,requests.api.post,requests.api.put,requests.api.request + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME, + XXX + +# Regular expression of note tags to take in consideration. +notes-rgx= + + +[REFACTORING] + +# Maximum number of nested blocks for function / method body +max-nested-blocks=5 + +# Complete name of functions that never returns. When checking for +# inconsistent-return-statements if a never returning function is called then +# it will be considered as an explicit return statement and no message will be +# printed. +never-returning-functions=sys.exit,argparse.parse_error + +# Let 'consider-using-join' be raised when the separator to join on would be +# non-empty (resulting in expected fixes of the type: ``"- " + " - +# ".join(items)``) +suggest-join-with-non-empty-separator=yes + + +[REPORTS] + +# Python expression which should return a score less than or equal to 10. You +# have access to the variables 'fatal', 'error', 'warning', 'refactor', +# 'convention', and 'info' which contain the number of messages in each +# category, as well as 'statement' which is the total number of statements +# analyzed. This score is used by the global evaluation report (RP0004). +evaluation=max(0, 0 if fatal else 10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10)) + +# Template used to display messages. This is a python new-style format string +# used to format the message information. See doc for all details. +msg-template= + +# Set the output format. Available formats are: 'text', 'parseable', +# 'colorized', 'json2' (improved json format), 'json' (old json format), msvs +# (visual studio) and 'github' (GitHub actions). You can also give a reporter +# class, e.g. mypackage.mymodule.MyReporterClass. +#output-format= + +# Tells whether to display a full report or only the messages. +reports=yes + +# Activate the evaluation score. +score=yes + + +[SIMILARITIES] + +# Comments are removed from the similarity computation +ignore-comments=yes + +# Docstrings are removed from the similarity computation +ignore-docstrings=yes + +# Imports are removed from the similarity computation +ignore-imports=yes + +# Signatures are removed from the similarity computation +ignore-signatures=yes + +# Minimum lines number of a similarity. +min-similarity-lines=4 + + +[SPELLING] + +# Limits count of emitted suggestions for spelling mistakes. +max-spelling-suggestions=4 + +# Spelling dictionary name. No available dictionaries : You need to install +# both the python package and the system dependency for enchant to work. +spelling-dict= + +# List of comma separated words that should be considered directives if they +# appear at the beginning of a comment and should not be checked. +spelling-ignore-comment-directives=fmt: on,fmt: off,noqa:,noqa,nosec,isort:skip,mypy: + +# List of comma separated words that should not be checked. +spelling-ignore-words= + +# A path to a file that contains the private dictionary; one word per line. +spelling-private-dict-file= + +# Tells whether to store unknown words to the private dictionary (see the +# --spelling-private-dict-file option) instead of raising a message. +spelling-store-unknown-words=no + + +[STRING] + +# This flag controls whether inconsistent-quotes generates a warning when the +# character used as a quote delimiter is used inconsistently within a module. +check-quote-consistency=no + +# This flag controls whether the implicit-str-concat should generate a warning +# on implicit string concatenation in sequences defined over several lines. +check-str-concat-over-line-jumps=no + + +[TYPECHECK] + +# List of decorators that produce context managers, such as +# contextlib.contextmanager. Add to this list to register other decorators that +# produce valid context managers. +contextmanager-decorators=contextlib.contextmanager + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E1101 when accessed. Python regular +# expressions are accepted. +generated-members= + +# Tells whether to warn about missing members when the owner of the attribute +# is inferred to be None. +ignore-none=yes + +# This flag controls whether pylint should warn about no-member and similar +# checks whenever an opaque object is returned when inferring. The inference +# can return multiple potential results while evaluating a Python object, but +# some branches might not be evaluated, which results in partial inference. In +# that case, it might be useful to still emit no-member and other checks for +# the rest of the inferred objects. +ignore-on-opaque-inference=yes + +# List of symbolic message names to ignore for Mixin members. +ignored-checks-for-mixins=no-member, + not-async-context-manager, + not-context-manager, + attribute-defined-outside-init + +# List of class names for which member attributes should not be checked (useful +# for classes with dynamically set attributes). This supports the use of +# qualified names. +ignored-classes=optparse.Values,thread._local,_thread._local,argparse.Namespace + +# Show a hint with possible names when a member name was not found. The aspect +# of finding the hint is based on edit distance. +missing-member-hint=yes + +# The maximum edit distance a name should have in order to be considered a +# similar match for a missing member name. +missing-member-hint-distance=1 + +# The total number of similar names that should be taken in consideration when +# showing a hint for a missing member. +missing-member-max-choices=1 + +# Regex pattern to define which classes are considered mixins. +mixin-class-rgx=.*[Mm]ixin + +# List of decorators that change the signature of a decorated function. +signature-mutators= + + +[VARIABLES] + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid defining new builtins when possible. +additional-builtins= + +# Tells whether unused global variables should be treated as a violation. +allow-global-unused-variables=yes + +# List of names allowed to shadow builtins +allowed-redefined-builtins= + +# List of strings which can identify a callback function by name. A callback +# name must start or end with one of those strings. +callbacks=cb_, + _cb + +# A regular expression matching the name of dummy variables (i.e. expected to +# not be used). +dummy-variables-rgx=_+$|(_[a-zA-Z0-9_]*[a-zA-Z0-9]+?$)|dummy|^ignored_|^unused_ + +# Argument names that match this expression will be ignored. +ignored-argument-names=_.*|^ignored_|^unused_ + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# List of qualified module names which can have objects that can redefine +# builtins. +redefining-builtins-modules=six.moves,past.builtins,future.builtins,builtins,io diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..48fa2a0 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,61 @@ +[build-system] +requires = ["setuptools"] +build-backend = "setuptools.build_meta" + +[project] +name = "fremorizer" +description = "Model output rewriter (CMORizer) for FRE/FMS based models" +readme = "README.md" +requires-python = ">=3.11" +license = "Apache-2.0" +authors = [ + {name = "MSD Workflow Team", email="oar.gfdl.workflow@noaa.gov"} +] + +keywords = [ + "gfdl", + "workflow", + "fre", + "fms", + "cmor", +] + +dependencies = [ + 'cftime', + 'click', + 'cmor', + 'netCDF4', + 'numpy', + 'pyyaml', +] + +classifiers = [ + "Environment :: Console", + "Operating System :: POSIX", + "Programming Language :: Python :: 3", +] + +dynamic = [ + "version", +] + + +[tool.setuptools.dynamic] +version = {attr = "fremorizer.__version__"} + + +[project.scripts] +fremor = 'fremorizer.fremor:fremor' + + +[project.optional-dependencies] +docs = [ + "sphinx", + "renku-sphinx-theme", + "sphinx-rtd-theme", +] + +test = [ + "pylint", + "pytest", +]