Merge branch 'master' into doc_update

Author: jasonb5

CIME/SystemTests/system_tests_common.py

@@ -211,9 +211,10 @@ def _set_restart_interval(
             smon = int(smon)
             sday = int(sday)
         else:
-            syr = int(startdate) / 10000
-            smon = int(startdate) - syr * 10000 / 100
-            sday = int(startdate) - syr * 10000 - smon * 100
+            startdate = int(startdate)
+            syr = int(startdate / 10000)
+            smon = int((startdate - syr * 10000) / 100)
+            sday = startdate - syr * 10000 - smon * 100
 
         addyr = syr // 10000
         syr = syr % 10000

CIME/Tools/Makefile

@@ -655,14 +655,6 @@ ULIBS += -L$(INSTALL_SHAREDPATH)/lib
 
 ULIBS += -lcsm_share -L$(INSTALL_SHAREDPATH)/lib $(PIOLIBNAME) -lgptl
 
-# Add Kokkos header files and libraries for SE-NH dycore
-ifdef USE_KOKKOS
-  CAM_TARGET ?= $(shell $(CASEROOT)/xmlquery --caseroot $(CASEROOT) CAM_TARGET --value)
-  ifeq ($(filter $(CAM_TARGET),theta-l theta-l_kokkos),$(CAM_TARGET))
-    ULIBS += -L$(INSTALL_SHAREDPATH)/lib64 -lkokkoscontainers -lkokkoscore -lkokkossimd
-  endif
-endif
-
 #------------------------------------------------------------------------------
 # Drive cmake script for cism and pio
 #------------------------------------------------------------------------------
@@ -738,6 +730,7 @@ CMAKE_ENV_VARS += CC=$(CC) \
 		  FC=$(FC) \
 		  LDFLAGS="$(LDFLAGS)"
 
+F90_LDFLAGS += $(FFLAGS)
 
 # We declare GLCMakefile to be a phony target so that cmake is
 # always rerun whenever invoking 'make GLCMakefile'; this is

CIME/XML/env_batch.py

@@ -237,21 +237,19 @@ def get_job_overrides(self, job, case):
         # when developed this variable was only needed on derecho, but I have tried to
         # make it general enough that it can be used on other systems by defining MEM_PER_TASK and MAX_MEM_PER_NODE in config_machines.xml
         # and adding {{ mem_per_node }} in config_batch.xml
-        try:
-            mem_per_task = case.get_value("MEM_PER_TASK")
-            max_mem_per_node = case.get_value("MAX_MEM_PER_NODE")
-            mem_per_node = total_tasks
-
-            if mem_per_node < mem_per_task:
-                mem_per_node = mem_per_task
-            elif mem_per_node > max_mem_per_node:
-                mem_per_node = max_mem_per_node
-            overrides["mem_per_node"] = mem_per_node
-        except TypeError:
-            # ignore this, the variables are not defined for this machine
-            pass
-        except Exception as error:
-            print("An exception occured:", error)
+        mem_per_task = case.get_value("MEM_PER_TASK")
+        max_tasks_per_node = case.get_value("MAX_TASKS_PER_NODE")
+        expect(
+            max_tasks_per_node > 0,
+            "Error MAX_TASKS_PER_NODE not set or set incorrectly",
+        )
+        max_mem_per_node = case.get_value("MAX_MEM_PER_NODE")
+        if mem_per_task and total_tasks <= max_tasks_per_node:
+            overrides["mem_per_node"] = int(
+                float(total_tasks) / float(max_tasks_per_node) * max_mem_per_node
+            )
+        elif max_mem_per_node:
+            overrides["mem_per_node"] = max_mem_per_node
 
         overrides["ngpus_per_node"] = ngpus_per_node
         overrides["mpirun"] = case.get_mpirun_cmd(job=job, overrides=overrides)

CIME/bless_test_results.py

@@ -9,7 +9,7 @@
 )
 from CIME.config import Config
 from CIME.test_status import *
-from CIME.hist_utils import generate_baseline, compare_baseline
+from CIME.hist_utils import generate_baseline, compare_baseline, NO_ORIGINAL
 from CIME.case import Case
 from CIME.test_utils import get_test_status_files
 from CIME.baselines.performance import (
@@ -180,6 +180,18 @@ def bless_history(test_name, case, baseline_name, baseline_root, report_only, fo
             if not report_only and (
                 force or input("Update this diff (y/n)? ").upper() in ["Y", "YES"]
             ):
+                # Sometimes, in order to get things passing, files have to be removed
+                # from the baseline area.
+                for line in cmp_comments.splitlines():
+                    if NO_ORIGINAL in line:
+                        file_to_remove = (
+                            line.split(NO_ORIGINAL)[0].split()[-1].strip("'")
+                        )
+                        logger.info(
+                            "Removing stale baseline file {}".format(file_to_remove)
+                        )
+                        os.remove(os.path.join(baseline_full_dir, file_to_remove))
+
                 gen_result, gen_comments = generate_baseline(
                     case, baseline_dir=baseline_full_dir
                 )

CIME/build.py

@@ -755,7 +755,7 @@ def _build_libraries(
     if mpilib == "mpi-serial":
         libs.insert(0, mpilib)
 
-    if uses_kokkos(case):
+    if uses_kokkos(case) and comp_interface != "nuopc":
         libs.append("kokkos")
 
     # Build shared code of CDEPS nuopc data models

CIME/build_scripts/buildlib.mpi-serial

@@ -80,10 +80,11 @@ def buildlib(bldroot, installpath, case):
     run_bld_cmd_ensure_logging(cmd, logger)
 
     # Now we run the mpi-serial make command
-    gmake_opts = "-f {} ".format(os.path.join(bldroot, "Makefile"))
+    gmake_opts = "-f {} ".format(os.path.join(mpi_serial_path, "Makefile"))
     gmake_opts += " -C {} ".format(bldroot)
     gmake_opts += " -j {} ".format(case.get_value("GMAKE_J"))
-    gmake_opts += " SRCDIR={} ".format(os.path.join(mpi_serial_path))
+    gmake_opts += " SRCDIR={} ".format(mpi_serial_path)
+    gmake_opts += " VPATH={}".format(mpi_serial_path)
 
     cmd = "{} {}".format(gmake_cmd, gmake_opts)
     run_bld_cmd_ensure_logging(cmd, logger)

CIME/case/case_st_archive.py

@@ -168,42 +168,59 @@ def _archive_rpointer_files(
     datename_is_last,
 ):
     ###############################################################################
-    if datename_is_last:
-        rpointers = glob.glob(
-            os.path.join(rundir, "rpointer.*" + _datetime_str(datename))
-        )
-        # Copy of all rpointer files for latest restart date
-        rpointers = glob.glob(os.path.join(rundir, "rpointer.*"))
-        for rpointer in rpointers:
-            safe_copy(
-                rpointer, os.path.join(archive_restdir, os.path.basename(rpointer))
-            )
-    else:
-        # Generate rpointer file(s) for interim restarts for the one datename and each
-        # possible value of ninst_strings
-        if save_interim_restart_files:
-            rpointers = glob.glob(
-                os.path.join(rundir, "rpointer.*" + _datetime_str(datename))
-            )
-            # If timestamped rpointers exist use them
-            if rpointers:
-                for rpointer in rpointers:
-                    safe_copy(
-                        rpointer,
-                        os.path.join(archive_restdir, os.path.basename(rpointer)),
-                    )
-            else:
-                # parse env_archive.xml to determine the rpointer files
-                # and contents for the given archive_entry tag
-                rpointer_items = archive.get_rpointer_contents(archive_entry)
 
-                # loop through the possible rpointer files and contents
-                for rpointer_file, rpointer_content in rpointer_items:
-                    temp_rpointer_file = rpointer_file
-                    temp_rpointer_content = rpointer_content
+    # parse env_archive.xml to determine the rpointer files
+    # and contents for the given archive_entry tag
+    # loop through the possible rpointer files and contents
+    rpointer_nodes = archive.get_children("rpointer", root=archive_entry)
+    for rpointer in rpointer_nodes:
+        file_node = archive.get_child("rpointer_file", root=rpointer)
+        temp_rpointer_file = archive.text(file_node)
+        content_node = archive.get_child("rpointer_content", root=rpointer)
+        temp_rpointer_content = archive.text(content_node)
+        rpointer_file = temp_rpointer_file.replace("$NINST_STRING", "*")
+        if rpointer_file == "unset":
+            continue
+        if "$DATENAME" in rpointer_file:
+            rpointer_file = rpointer_file.replace("$DATENAME", _datetime_str(datename))
+
+        expect(
+            not "$" in rpointer_file,
+            "Unrecognized expression in name {}".format(rpointer_file),
+        )
+        rpointers = glob.glob(rundir + "/" + rpointer_file)
+        if datename_is_last:
+            for rpfile in rpointers:
+                safe_copy(
+                    rpfile, os.path.join(archive_restdir, os.path.basename(rpfile))
+                )
+        else:
+            # Generate rpointer file(s) for interim restarts for the one datename and each
+            # possible value of ninst_strings
+            if save_interim_restart_files:
+                # If timestamped rpointers exist use them
+                if rpointers:
+                    for rpfile in rpointers:
+                        logger.info("moving interim rpointer_file {}".format(rpfile))
+                        shutil.move(
+                            rpfile,
+                            os.path.join(archive_restdir, os.path.basename(rpfile)),
+                        )
+                else:
 
                     # put in a temporary setting for ninst_strings if they are empty
                     # in order to have just one loop over ninst_strings below
+                    if ninst_strings:
+                        rpointer_content = temp_rpointer_content.replace(
+                            "$NINST_STRING", ninst_strings[0]
+                        )
+                    else:
+                        rpointer_content = temp_rpointer_content.replace(
+                            "$NINST_STRING", ""
+                        )
+                    rpointer_content = rpointer_content.replace(
+                        "$DATENAME", _datetime_str(datename)
+                    )
                     if rpointer_content != "unset":
                         if not ninst_strings:
                             ninst_strings = ["empty"]

CIME/case/case_submit.py

@@ -287,7 +287,7 @@ def submit(
             functor,
             "case.submit",
             caseroot=caseroot,
-            custom_success_msg_functor=lambda x: x.split(":")[-1],
+            custom_success_msg_functor=lambda x: x,
             is_batch=is_batch,
             gitinterface=self._gitinterface,
         )

CIME/data/config/cesm/config_files.xml

@@ -647,7 +647,6 @@
   <entry id="BUILD_LIB_FILE">
     <type>char</type>
     <values>
-      <value lib="kokkos">$SRCROOT/libraries/build_scripts/buildlib.kokkos</value>
       <value lib="FMS">$SRCROOT/libraries/FMS/buildlib</value>
       <value lib="CDEPS">$SRCROOT/components/cdeps/cime_config/buildlib</value>
       <value lib="CMEPS">$SRCROOT/components/cmeps/cime_config/buildlib</value>

CIME/non_py/src/timing/perf_mod.F90

@@ -1162,7 +1162,7 @@ subroutine t_prf(filename, mpicom, num_outpe, stride_outpe, &
 !-----------------------------------------------------------------------
 !
    if (.not. timing_initialized) return
-#ifdef NUOPC_INTERFACE
+#if defined NUOPC_INTERFACE && ! (defined MODEL_THETA_L && defined HOMME_ENABLE_COMPOSE)
    return
 #endif