Ready for trial run

Author: Steve Goldhaber

tools/cmip_namelists/README.md

@@ -21,7 +21,7 @@ variables, e.g., `nhtfrq`.
 
 To run the script, first download a current version of the [CMIP7 data
 request
-spreadsheet](https://docs.google.com/spreadsheets/d/1XUdCTl1zKsWi_yTvMZqsMtBnnUPIhIeTPVSLHgL0Hdo) and the separate [CAM field request spreadsheet]()
+spreadsheet](https://docs.google.com/spreadsheets/d/1XUdCTl1zKsWi_yTvMZqsMtBnnUPIhIeTPVSLHgL0Hdo) and the separate [CAM field request spreadsheet](https://docs.google.com/spreadsheets/d/1-ohp9nwJ5BUKQ7qxubg_PD1bVMWMMz00bS-MQ-z30p8/edit?gid=0#gid=0)
 (File `==>` Download `==>` .csv). The script is then run as:
 
 ```
@@ -32,3 +32,9 @@ For the full interface (with options), see the help menu:
 ```
 usage: cmip_diagnostic_namelists.py [--help]
 ```
+
+To use this script, the following workflow is recommended:
+1. Use `git rm` to remove existing usermods files (or use `--overwrite` and only remove files no longer in use).
+2. Run script to generate new usermods files
+3. Use `git add` to add any new usermods files
+4. Use `git commit -a` to commit all the new usermods files. Note that git will only show untracked directory names if none of the files in that directory are currently in the repository.

tools/cmip_namelists/cmip_diagnostic_namelists.py

@@ -35,10 +35,11 @@
 class Usermod():
     """Class to hold information about a history usermod directory"""
 
-    def __init__(self, name, frequencies, dependencies=None,
-                 include_cosp=False, include_aerocom=False):
+    def __init__(self, name, dirname, frequencies, usermods_dir,
+                 dependencies=None, include_cosp=False, include_aerocom=False):
         """Initialize a history usermod section"""
         self.__name = name
+        self.__dirname = os.path.normpath(os.path.join(usermods_dir, dirname))
         self.__freqset = set([x.strip() for x in frequencies.split(',')])
         if dependencies:
             self.__depends = [x.strip() for x in dependencies.split(',')]
@@ -50,27 +51,32 @@ def __init__(self, name, frequencies, dependencies=None,
 
     def unique_frequencies(self, usermod_dict):
         """Return the output frequencies not covered by this section's dependencies.
-        It is an error if any dependency is not in usermod_dict.
-        It is an error if any frequency in a dependency is not in this section's dependencies."""
+        """
         my_freqs = self.frequencies
         for dependency in self.dependencies:
-            if dependency in usermod_dict:
-                depmod = usermod_dict[dependency]
-                if depmod.frequencies - self.frequencies:
-                    raise ValueError(f"Section, '{dependency}' has frequencies not in '{self.name}'")
-                # end if
-                my_freqs -= depmod.frequencies
-            else:
-                raise ValueError(f"Dependency, '{dependency}', from {self.name}, not found")
-            # end if
+            depmod = usermod_dict[dependency]
+            my_freqs -= depmod.frequencies
         # end for
-        return my_freqs
+        return sorted(my_freqs, key=lambda x: _HIST_FILEORDER.index(x))
+
+    def namelist_file(self):
+        """Construct and return the namelist filename for this object"""
+        return os.path.join(self.dirname, "user_nl_cam")
+
+    def include_file(self):
+        """Construct and return the include usermods filename for this object"""
+        return os.path.join(self.dirname, "include_user_mods")
 
     @property
     def name(self):
         """Return the name for this object"""
         return self.__name
 
+    @property
+    def dirname(self):
+        """Return the usermod subdirectory name for this object"""
+        return self.__dirname
+
     @property
     def frequencies(self):
         """Return the frequencies for this object"""
@@ -127,13 +133,17 @@ def command_line(args):
                         help="""Stop processing if any missing fields found.
                         Default is to produce fieldlist files by ignoring any
                         missing fields.""")
+    parser.add_argument("--max-line", type=int, default=80,
+                        help="Maximum line length for namelist files")
     pargs = parser.parse_args(args)
     return (pargs.CMIP_file, pargs.CAM_file, pargs.usermods, pargs.cfgfile,
-            pargs.overwrite, pargs.error_on_missing)
+            pargs.overwrite, pargs.error_on_missing, pargs.max_line)
 
-def read_config_file(filename):
+def read_config_file(filename, usermods_dir, overwrite):
     """Read a fincl group configuration (ini-style) file.
-    Returns a dictionary of usermods sections with the section name as the key"""
+    Returns a dictionary of usermods sections with the section name as the key.
+    If any errors are found, print and return None"""
+    errors = False
     usermod_dict = {}
     config = configparser.ConfigParser()
     config.read(filename)
@@ -143,6 +153,7 @@ def read_config_file(filename):
         use_cosp = None
         include_aerocom = None
         frequencies = cfg_sect['frequencies']
+        dirname = cfg_sect['usermod_dir']
         if 'include_usermods' in cfg_sect:
             depends = cfg_sect['include_usermods']
         # end if
@@ -152,29 +163,66 @@ def read_config_file(filename):
         if 'use_aercom' in cfg_sect:
             include_aerocom = cfg_sect['use_aercom']
         # end if
-        if section in usermod_dict:
-            raise ValueError("Duplicate section, '{section}'")
-        # end if
-        usermod_dict[section] = Usermod(section, frequencies,
+        usermod_dict[section] = Usermod(section, dirname, frequencies,
+                                        usermods_dir,
                                         dependencies=depends,
                                         include_cosp=use_cosp,
                                         include_aerocom=include_aerocom)
     # end for
-    return usermod_dict
+    # Check for errors
+    for name, usermod in usermod_dict.items():
+        if name in usermod_dict:
+            print(f"Duplicate section, '{name}'")
+            errors = True
+        # end if
+        if not overwrite:
+            path = usermod.namelist_file()
+            if os.path.exists(path):
+                print(f"namelist file, '{path}' exists and overwrite = False")
+                errors = True
+            # end if
+        # end if
+        if not overwrite and usermod.dependencies:
+            path = usermod.include_file()
+            if os.path.exists(path):
+                print(f"include file, '{path}' exists and overwrite = False")
+                errors = True
+            # end if
+        # end if
+        for depend in usermod.dependencies:
+            if depend == usermod.name:
+                print(f"Invalid dependency, {depend} in section {depend}")
+                errors = True
+            # end if
+            # It is an error if any dependency is not in usermod_dict.
+            if depend not in usermod_dict:
+                print(f"Invalid dependency, '{depend}' not in config file")
+                errors = True
+            # end if
+        # end for
+        # Check frequencies
+        if not usermod.frequencies:
+            print(f"Section, '{name}', contains no output frequencies")
+            errors = True
+        elif if any([x not in _HIST_FILEORDER for x in usermod.frequencies]):
+            unknown = list(set(usermod.frequencies) - set(_HIST_FILEORDER))
+            freq = ', '.join(unknown)
+            print(f"Section, '{section}', contains unknown frequencies: {freq}")
+            errors = True
+        else:
+            for dependency in self.dependencies:
+                depmod = usermod_dict[dependency]
+                if depmod.frequencies - usermod.frequencies:
+                    print(f"Section, '{dependency}' has frequencies not in '{name}'")
+                    errors = True
+                # end if
 
-@contextlib.contextmanager
-def flex_open(filename=None, mode='w'):
-    if filename and filename != '-':
-        fh = open(filename, mode)
-    else:
-        fh = sys.stdout
+        # end if
+    # end for
+    if errors:
+        return None
     # end if
-
-    try:
-        yield fh
-    finally:
-        if fh is not sys.stdout:
-            fh.close()
+    return usermod_dict
 
 def read_fieldname_file(filename):
     """Read a fieldname file and return all fieldnames as a list."""
@@ -260,10 +308,11 @@ def parse_spreadsheet(csvfile, model_name="atmos"):
     # end for
     return cmip_dict
 
-def check_for_missing_fieldnames(masterlist, data_request):
+def check_for_missing_fieldnames(masterlist, data_request, request_name):
     """Given a data request dictionary (<data_request>),
     check to see if any are not in <masterlist>.
-    Return a list of missing fields names (an empty list means none).
+    Return a set of missing fields names (an empty list means none).
+    Print out any missing fields.
     Clean <data_request> to remove missing field entries (side effect)."""
     # Gather the set of all fields (combine different frequencies)
     all_reqfields = set()
@@ -281,65 +330,86 @@ def check_for_missing_fieldnames(masterlist, data_request):
         for field in sorted(missing):
             print(f"  {field}")
         # end for
+        print(f"These fields were found in the {request_name} data request spreadsheet")
     # end if
     return missing
 
-def generate_namelist_entries(data_request, nl_filename, maxline=125, hist_files=_HIST_FILEORDER):
-    """Write the set of namelist entries represented by <data_request> to
-    <nl_filename> (which may be standard output)."""
-    lbreak = ''
-    with flex_open(nl_filename, mode="w") as outfile:
-        for index, freq in enumerate(hist_files):
-            if freq in data_request:
-                if freq == 'subhr':
-                    avgflag = 'I'
-                else:
-                    avgflag = 'A'
-                # end if
-                # Write history file config info
-                outfile.write(f"{lbreak}{_HIST_TITLES[freq]}\n")
-                outfile.write(f"nhtfrq({index + 1}) = {_HIST_FRQCODES[freq]}\n")
-                outfile.write(f"mfilt({index + 1}) = {_HIST_MFILT[freq]}\n")
-                outfile.write(f"empty_htapes({index + 1}) = .true.\n")
-                fields = data_request[freq]
-                fldstring = ', '.join([f"{x}:{avgflag}" for x in fields])
-                nlstr = f"fincl{index + 1} = {fldstring}"
-                # Write the fincl string with appropriate line breaks
-                begpos = 0
-                strlen = len(nlstr)
-                while begpos < strlen:
-                    endpos = strlen
-                    if endpos - begpos > maxline:
-                        endpos = nlstr[0:begpos + maxline].rfind(' ')
-                        if endpos < begpos:
-                            endpos = strlen
-                        # end if
+def generate_namelist_entries(data_request, usermod_config, maxline):
+    """Write the sets of namelist entries represented by <data_request> to
+    the usermods files defined in <usermod_config>."""
+    for usermod in usermod_config.values():
+        lbreak = ''
+        if not os.path.exists(usemod.dirname):
+            os.makedirs(usemod.dirname)
+        # end if
+        with open(usermod.namelist_file(), mode="w") as outfile:
+            for freq in usermod.unique_frequencies(usermod_config)
+                if freq in data_request:
+                    # index is the fincl number for this frequency
+                    index = _HIST_FILEORDER.index(freq) + 1
+                    if freq == 'subhr':
+                        avgflag = 'I'
+                    else:
+                        avgflag = 'A'
                     # end if
-                    outfile.write(f"{nlstr[begpos:endpos]}\n")
-                    begpos = endpos
-                # end while
+                    # Write history file config info
+                    outfile.write(f"{lbreak}{_HIST_TITLES[freq]}\n")
+                    outfile.write(f"nhtfrq({index}) = {_HIST_FRQCODES[freq]}\n")
+                    outfile.write(f"mfilt({index}) = {_HIST_MFILT[freq]}\n")
+                    outfile.write(f"empty_htapes({index}) = .true.\n")
+                    fields = data_request[freq]
+                    fldstring = ', '.join([f"{x}:{avgflag}" for x in fields])
+                    nlstr = f"fincl{index} = {fldstring}"
+                    # Write the fincl string with appropriate line breaks
+                    begpos = 0
+                    strlen = len(nlstr)
+                    while begpos < strlen:
+                        endpos = strlen
+                        if endpos - begpos > maxline:
+                            endpos = nlstr[0:begpos + maxline].rfind(' ')
+                            if endpos < begpos:
+                                endpos = strlen
+                            # end if
+                        # end if
+                        outfile.write(f"{nlstr[begpos:endpos]}\n")
+                        begpos = endpos
+                    # end while
+                # end if
+                lbreak = '\n'
+                # end for
             # end if
-            lbreak = '\n'
-        # end for
-        # Now, write out combined namelist items
-    # end with
-
+        # end with (open file)
+        # Write the include usermods file (if any)
+        if usermod.dependencies:
+            with open(username.include_file, mode="w") as outfile:
+                for depend in usermod.dependencies:
+                    depdir = os.path.basename(usermod_config[depend].dirname)
+                    outfile.write(f"{os.path.join(os.pardir, depdir)}\n")
+                # end for
+            # end for
+        # end if
+    # end for (sections)
 
 ###############################################################################
 
 if __name__ == "__main__":
     arglist = command_line(sys.argv[1:])
-    cmipfile, camfile, usermods_dir, configfile, overwrite, error = arglist
+    cmipfile, camfile, usermods, configfile, overwrite, error, maxline = arglist
     # read configuration
-    usermod_dict = read_config_file(configfile)
-    all_fieldnames, cosp_fieldnames, aerocom_fieldnames = read_diagnostic_fieldnames()
-    cmip7_request = parse_spreadsheet(cmipfile)
-    missing7 = check_for_missing_fieldnames(all_fieldnames, cmip7_request, "CMIP7")
-    cam_request = parse_spreadsheet(camfile)
-    missingc = check_for_missing_fieldnames(all_fieldnames, cmip7_request, "CAM")
-    if error and (missing7 or missingc):
-        print("Missing fields found, not producing any namelist usermods files")
-#    else:
-#        generate_namelist_entries(data_request, , maxline=80)
+    usermod_dict = read_config_file(configfile, usermods, overwrite)
+    if usermod_dict:
+        fieldnames = read_diagnostic_fieldnames()
+        all_fieldnames, cosp_fieldnames, aerocom_fieldnames = fieldnames
+        cmip7_request = parse_spreadsheet(cmipfile)
+        missing7 = check_for_missing_fieldnames(all_fieldnames, cmip7_request,
+                                                "CMIP7")
+        cam_request = parse_spreadsheet(camfile)
+        missingc = check_for_missing_fieldnames(all_fieldnames, cmip7_request,
+                                                "CAM")
+        if error and (missing7 or missingc):
+            print("Missing fields found, not producing any namelist usermods files")
+        else:
+            generate_namelist_entries(data_request, usermod_dict, maxline)
+        # end if
     # end if
     sys.exit(0)

tools/cmip_namelists/usermods_sets.cfg

@@ -3,7 +3,10 @@
 ## Frequencies and usermod_dir are required
 ## Frequencies are mon, day, 6hr, 3hr, 1hr, subhr
 
-## include_usermods is the name of a config section (default: none)
+## include_usermods is a comma-separated list of names of config sections
+##  (default: none)
+## Each config section must be present in this file
+
 ## COSP_on and use_aerocom default to False
 
 [reduced]