Merge pull request #333 from xylar/write-cdf5

Switch to writing files in CDF5 (`NETCDF3_64BIT_DATA`) format

Author: xylar

deploy/default.cfg

@@ -23,15 +23,15 @@ mpi = nompi
 geometric_features = 1.6.1
 mache = 1.31.0
 conda_moab = 5.5.1
-mpas_tools = 1.1.0
+mpas_tools = 1.2.0
 otps = 2021.10
 parallelio = 2.6.6
 
 # versions of conda or spack packages (depending on machine type)
 esmf = 8.8.1
 metis = 5.1.0
 netcdf_c = 4.9.2
-netcdf_fortran = 4.6.1
+netcdf_fortran = 4.6.2
 pnetcdf = 1.14.0
 
 # versions of spack packages

docs/developers_guide/framework/model.md

@@ -430,3 +430,97 @@ call {py:func}`polaris.model_step.make_graph_file()` to produce a graph file
 from an MPAS mesh file.  Optionally, you can provide the name of an MPAS field
 on cells in the mesh file that gives different weight to different cells
 (`weight_field`) in the partitioning process.
+
+## Detailed Documentation: polaris.namelist, polaris.streams, and polaris.yaml
+
+### `polaris.namelist`
+
+This module provides utilities for reading, parsing, updating, and writing
+MPAS-style namelist files. It is used to manage namelist options for E3SM
+components.
+
+**Key Functions:**
+- `parse_replacements(package, namelist)`: Reads a namelist file from a
+  package and returns a dictionary of option replacements.
+- `ingest(defaults_filename)`: Reads a full namelist file and returns a nested
+  dictionary of records and options.
+- `replace(namelist, replacements)`: Applies replacements to a namelist
+  dictionary.
+- `write(namelist, filename)`: Writes a nested namelist dictionary to a file.
+
+**Example Usage:**
+```python
+from polaris import namelist
+
+replacements = namelist.parse_replacements('my_package', 'namelist.forward')
+namelist_dict = namelist.ingest('namelist.defaults')
+updated = namelist.replace(namelist_dict, replacements)
+namelist.write(updated, 'namelist.input')
+```
+
+### `polaris.streams`
+
+This module provides tools for reading, updating, and writing MPAS streams XML
+files, including support for Jinja2 templating.
+
+**Key Functions:**
+- `read(package, streams_filename, tree=None, replacements=None)`: Reads a
+  streams XML file (optionally as a Jinja2 template) and returns an XML tree.
+- `write(streams, out_filename)`: Writes a streams XML tree to a file.
+- `update_tree(tree, new_tree)`: Updates an existing streams tree with new
+  streams.
+- `set_default_io_type(tree, io_type='pnetcdf,cdf5')`: Sets the `io_type`
+  attribute for output streams if not already set.
+
+**Example Usage:**
+```python
+from polaris import streams
+
+tree = streams.read('my_package', 'streams.forward')
+streams.write(tree, 'streams.xml')
+```
+
+### `polaris.yaml`
+
+This module provides a class for reading, writing, and updating YAML
+configuration files for E3SM components, including support for Jinja2
+templating and conversion between YAML and MPAS namelist/streams formats.
+
+**Key Classes and Functions:**
+- `PolarisYaml`: Main class for managing YAML configs.
+  - `PolarisYaml.read(filename, package=None, replacements=None, ...)`: Reads
+    a YAML file (optionally as a Jinja2 template).
+  - `PolarisYaml.update(configs=None, options=None, quiet=True)`: Updates the
+    config with new options.
+  - `PolarisYaml.write(filename)`: Writes the config to a YAML file.
+- `mpas_namelist_and_streams_to_yaml(...)`: Converts MPAS namelist and streams
+  files to a YAML config.
+
+**Example Usage:**
+```python
+from polaris.yaml import PolarisYaml
+
+yaml_cfg = PolarisYaml.read('config.yaml')
+yaml_cfg.update(options={'config_dt': '00:10:00'})
+yaml_cfg.write('updated_config.yaml')
+```
+
+**See also:**
+Refer to the API documentation for each module for a full list of functions
+and classes:
+* {py:func}`polaris.namelist.parse_replacements`
+* {py:func}`polaris.namelist.ingest`
+* {py:func}`polaris.namelist.replace`
+* {py:func}`polaris.namelist.write`
+* {py:func}`polaris.streams.read`
+* {py:func}`polaris.streams.write`
+* {py:func}`polaris.streams.update_defaults`
+* {py:func}`polaris.streams.update_tree`
+* {py:func}`polaris.streams.set_default_io_type`
+* {py:class}`polaris.yaml.PolarisYaml`
+* {py:meth}`polaris.yaml.PolarisYaml.read`
+* {py:meth}`polaris.yaml.PolarisYaml.update`
+* {py:meth}`polaris.yaml.PolarisYaml.write`
+* {py:func}`polaris.yaml.mpas_namelist_and_streams_to_yaml`
+* {py:func}`polaris.yaml.yaml_to_mpas_streams`
+* {py:func}`polaris.yaml.main_mpas_to_yaml`

docs/developers_guide/organization/steps.md

@@ -20,7 +20,7 @@ Some steps are shared between multiple tasks to avoid redundant computation and
 to ensure consistency of results. Shared steps are created at the highest
 directory level common to all tasks that use them. To facilitate this, the
 `Component` class provides the method
-{py:method}`polaris.Component.get_or_create_shared_step()`.
+{py:meth}`polaris.Component.get_or_create_shared_step()`.
 
 This function checks if a step already exists in a component's `steps`
 dictionary under a given subdirectory. If it does, it returns the existing

docs/users_guide/ocean/tasks/overflow.md

@@ -3,9 +3,7 @@
 # overflow
 
 The ``ocean/overflow`` test group induces a density current flowing down a
-continental slope and includes two test cases. 
-
-(ocean-category-of-task-task-name)=
+continental slope and includes two test cases.
 
 ## suppported models
 
@@ -31,7 +29,7 @@ The mesh is planar and the resolution is specified by config option
 `overflow:resolution`, which defaults to 1 km.
 
 The horizontal dimensions of the domain are set by config options
-`overflow:lx` and `overflow:ly`, defaulting to 200 km by 40 km. 
+`overflow:lx` and `overflow:ly`, defaulting to 200 km by 40 km.
 
 The domain is periodic on the zonal boundaries and solid on the meridional
 boundaries.

docs/users_guide/ocean/tasks/template.md

@@ -4,8 +4,6 @@
 
 Description of common characteristics of the tasks.
 
-(ocean-category-of-task-task-name)=
-
 ## suppported models
 
 This section should have one of the following lines, as appropriate, or if
@@ -18,6 +16,8 @@ These tasks support only MPAS-Ocean.
 
 These tasks support only Omega.
 
+(ocean-category-of-task-task-name)=
+
 ## task_name
 
 In cases where the test cases within a category share many characteristics,

polaris/default.cfg

@@ -36,14 +36,11 @@ login_cores = 4
 [io]
 
 # the NetCDF file format: NETCDF4, NETCDF4_CLASSIC, NETCDF3_64BIT, or
-# NETCDF3_CLASSIC
-format = NETCDF3_64BIT
-
-# the NetCDF output engine: netcdf4 or scipy
-# the netcdf4 engine is not performing well on Chrysalis and Anvil, so we will
-# try scipy for now.  If we can switch to NETCDF4 format, netcdf4 will be
-# required
-engine = scipy
+# NETCDF3_CLASSIC, NETCDF3_64BIT_DATA
+format = NETCDF3_64BIT_DATA
+
+# the NetCDF output engine: netcdf4, h5netcdf or scipy
+engine = netcdf4
 
 
 # Config options related to creating a job script

polaris/mesh/spherical.py

@@ -3,9 +3,10 @@
 import jigsawpy
 import matplotlib.pyplot as plt
 import numpy as np
-import xarray
+import xarray as xr
 from jigsawpy.savejig import savejig
 from mpas_tools.cime.constants import constants
+from mpas_tools.io import write_netcdf
 from mpas_tools.logging import check_call
 from mpas_tools.mesh.creation.jigsaw_to_netcdf import jigsaw_to_netcdf
 from mpas_tools.ocean.inject_meshDensity import inject_spherical_meshDensity
@@ -70,7 +71,7 @@ def save_and_plot_cell_width(self, lon, lat, cell_width):
             m x n array of cell width in km
         """
         section = self.config['spherical_mesh']
-        da = xarray.DataArray(
+        da = xr.DataArray(
             cell_width,
             dims=['lat', 'lon'],
             coords={'lat': lat, 'lon': lon},
@@ -118,12 +119,17 @@ def run(self):
 
         logger.info('Convert from triangles to MPAS mesh')
 
-        args = ['MpasMeshConverter.x', 'mesh_triangles.nc', mpas_mesh_filename]
+        tmp_mesh_filename = 'mpas_mesh_netcdf4.nc'
+        args = ['MpasMeshConverter.x', 'mesh_triangles.nc', tmp_mesh_filename]
         check_call(args=args, logger=logger)
 
+        # open the mesh and rewrite it in the desired NetCDF format
+        ds_mesh = xr.open_dataset(tmp_mesh_filename)
+        write_netcdf(ds_mesh, mpas_mesh_filename)
+
         if section.getboolean('add_mesh_density'):
             logger.info('Add meshDensity into the mesh file')
-            ds = xarray.open_dataset('cellWidthVsLatLon.nc')
+            ds = xr.open_dataset('cellWidthVsLatLon.nc')
             inject_spherical_meshDensity(
                 ds.cellWidth.values,
                 ds.lon.values,

polaris/model_step.py

@@ -699,6 +699,9 @@ def _write_model_config(self):
                     'Trying to write a streams file but no '
                     'streams XML tree was created.'
                 )
+            io_format = self.config.get('io', 'format')
+            if io_format == 'NETCDF3_64BIT_DATA':
+                polaris.streams.set_default_io_type(self._streams_tree)
             polaris.streams.write(self._streams_tree, streams_filename)
         # set these back to None because we don't need to keep them around
         # and the streams tree can't be pickled

polaris/streams.py

@@ -197,3 +197,18 @@ def _update_element(new_child, elements):
     if not found:
         # add a deep copy of the element
         elements.append(deepcopy(new_child))
+
+
+def set_default_io_type(tree, io_type='pnetcdf,cdf5'):
+    """
+    Set io_type attribute for all <stream> and <immutable_stream>
+    elements if not already set, except for immutable_stream with name 'mesh'.
+    """
+    streams = next(tree.iter('streams'))
+    all_streams = streams.findall('stream') + streams.findall(
+        'immutable_stream'
+    )
+    for stream in all_streams:
+        stream_type = stream.attrib.get('type')
+        if 'io_type' not in stream.attrib and 'output' in stream_type:
+            stream.attrib['io_type'] = io_type

polaris/version.py

@@ -1 +1 @@
-__version__ = '0.8.0-alpha.1'
+__version__ = '0.8.0-alpha.2'