Merge branch 'main' into add/tacc_vista

Author: jedwards4b

component_grids_mct.xml

@@ -257,6 +257,13 @@
     <support>Test support only</support>
   </domain>
 
+  <domain name="ne0np4.NATL.ne30x8">
+    <nx>317567</nx> <ny>1</ny>
+    <mesh>$DIN_LOC_ROOT/share/meshes/NATL_ne30x8_np4_MESH_cdf5_c20250618.nc</mesh>
+    <desc>ne0np4.NATL.ne30x8 is a Spectral Elem 1-deg grid with a 1/8 deg refined region over the North Atlantic:</desc>
+    <support>Test support only</support>
+  </domain>
+
   <!-- CAM/MPAS meshes -->
   <domain name="mpasa480">
     <nx>2562</nx> <ny>1</ny>
@@ -397,6 +404,67 @@
     <desc>tripole v1 1-deg grid: testing proxy for high-res tripole ocean grids- do not use for scientific experiments</desc>
     <support>Experimental tripole ocean grid</support>
   </domain>
+    <domain name="oQU480">
+      <nx>7234</nx>  <ny>1</ny>
+      <mesh driver="nuopc">$DIN_LOC_ROOT/ocn/mpas-o/oQU480/oQU480_ESMFmesh.nc</mesh>
+      <desc>oQU480 is a MPAS ocean grid that is roughly 4 degree resolution:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+    <domain name="oQU240">
+      <nx>7234</nx>  <ny>1</ny>
+      <mesh driver="nuopc">$DIN_LOC_ROOT/ocn/mpas-o/oQU240/oQU240_ESMFmesh.nc</mesh>
+      <desc>oQU240 is a MPAS ocean grid that is roughly 2 degree resolution:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+    <domain name="oQU120">
+      <nx>28876</nx>  <ny>1</ny>
+      <mesh driver="nuopc">$DIN_LOC_ROOT/ocn/mpas-o/oQU120/oQU120_ESMFmesh.230524.nc</mesh>
+      <desc>oQU120 is a MPAS ocean grid that is roughly 1 degree resolution:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+    <domain name="oEC60to30v3">
+      <nx>28571</nx>  <ny>1</ny>
+      <mesh driver="nuopc">/glade/work/dazlich/ec60_30/oEC60to30_ESMFmesh.nc</mesh>
+      <desc>oEC60to30v3 is a MPAS ocean grid that is roughly 1/2 degree resolution:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+    <domain name="oQU060">
+      <nx>115494</nx>  <ny>1</ny>
+      <mesh driver="nuopc">$DIN_LOC_ROOT/ocn/mpas-o/oQU060/oQU060_ESMFmesh.230907.nc</mesh>
+      <desc>oQU060 is a MPAS ocean grid that is roughly 1/2 degree resolution:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+    <domain name="oQU030">
+      <nx>462000</nx>  <ny>1</ny>
+      <mesh driver="nuopc">$DIN_LOC_ROOT/ocn/mpas-o/oQU030/oQU030_ESMFmesh.241221.nc</mesh>
+      <desc>oQU030 is a MPAS ocean grid that is roughly 1/4 degree resolution:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+    <domain name="oQU015">
+      <nx>1850090</nx>  <ny>1</ny>
+      <mesh driver="nuopc">$DIN_LOC_ROOT/ocn/mpas-o/oQU015/oQU015_ESMFmesh.241221.nc</mesh>
+      <desc>oQU015 is a MPAS ocean grid that is roughly 1/8 degree resolution:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+    <domain name="oQU0075">
+      <nx>7399783</nx>  <ny>1</ny>
+      <mesh driver="nuopc">$DIN_LOC_ROOT/ocn/mpas-o/oQU0075/oQU0075_ESMFmesh.241221.nc</mesh>
+      <desc>oQU0075 is a MPAS ocean grid that is roughly 1/16 degree resolution:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+    <domain name="oQU00375">
+      <nx>29598824</nx>  <ny>1</ny>
+      <mesh driver="nuopc">$DIN_LOC_ROOT/ocn/mpas-o/oQU00375/oQU00375_ESMFmesh.241221.nc</mesh>
+      <desc>oQU00375 is a MPAS ocean grid that is roughly 1/32 degree resolution:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+    <domain name="oRR60-3WUS">
+      <nx>835536</nx>  <ny>1</ny>
+      <mesh driver="nuopc">/glade/work/dazlich/oRR60-3/oRR60-3WUS_ESMFmesh.nc</mesh>
+      <desc>oQU60-3wus is a MPAS ocean grid that s 3km region refinement over the western US:</desc>
+      <support>Experimental, under development</support>
+    </domain>
+
 
   <!-- ======================================================== -->
   <!-- ROF domains -->
@@ -528,7 +596,7 @@
   </domain>
   <domain name="wgx3v7">
     <nx>100</nx>  <ny>116</ny>
-    <mesh>$DIN_LOC_ROOT/share/meshes/wgx3v7_2301208_ESMFmesh.nc</mesh>
+    <mesh>$DIN_LOC_ROOT/share/meshes/wgx3v7_240327_ESMFmesh.nc</mesh>
     <desc>gx3v7 global grid</desc>
     <support>For testing of the WAV model</support>
   </domain>

component_grids_nuopc.xml

@@ -7,35 +7,28 @@
   <!-- inputdata_chksum.dat is found on the server in the directory above inputdata -->
   <!-- it will be searched for filename and chksum of each downloaded file.  -->
   <!-- see the file ftp://ftp.cgd.ucar.edu/cesm/inputdata_chksum.dat for proper format. -->
-  <server>
-    <comment>grid ftp requires the globus-url-copy tool on the client side </comment>
-    <protocol>gftp</protocol>
-    <address>ftp://gridanon.cgd.ucar.edu:2811/cesm/inputdata/</address>
-    <checksum>../inputdata_checksum.dat</checksum>
-  </server>
-
   <server>
     <protocol>wget</protocol>
-    <address>ftp://ftp.cgd.ucar.edu/cesm/inputdata/</address>
-    <user>anonymous</user>
-    <password>[email protected]</password>
+    <address>https://ftp.cgd.ucar.edu/cesm/inputdata/</address>
     <checksum>../inputdata_checksum.dat</checksum>
   </server>
 
   <server>
-    <comment> ftp requires the python package ftplib </comment>
-    <protocol>ftp</protocol>
-    <address>ftp.cgd.ucar.edu/cesm/inputdata</address>
-    <user>anonymous</user>
-    <password>[email protected]</password>
-    <checksum>../inputdata_checksum.dat</checksum>
+    <comment> wget access to subversion </comment>
+    <protocol>wget</protocol>
+    <address>https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata/</address>
   </server>
 
   <server>
     <protocol>svn</protocol>
     <address>https://svn-ccsm-inputdata.cgd.ucar.edu/trunk/inputdata</address>
   </server>
 
+  <server>
+    <protocol>wget</protocol>
+    <address>https://redoak.cs.toronto.edu/twitcher/ows/proxy/thredds/fileServer/datasets/CESM/inputdata/</address>
+  </server>
+  
   <server CLM_USRDAT_NAME="NEON">
     <comment> NEON Tower data for datm </comment>
     <protocol>wget</protocol>

config_grids_mct.xml

@@ -14,3 +14,12 @@ mo_fluxes_broadband_kernels.o
 
 $(RRTMGP_OBJS): %.o: %.F90
 	$(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fno-openmp $<
+
+ifeq ($(COMP_NAME),mpassi)
+# mpas seaice files that can't compile with optimization using gnu
+MPASSI_NOOPT_OBJS=\
+ice_shortwave.o
+
+$(MPASSI_NOOPT_OBJS): %.o : %.F90
+	$(FC) -c $(FPPFLAGS) $(INCLDIR) $(INCS) $(FREEFLAGS) $(FFLAGS) $(FFLAGS_NOOPT) $<
+endif

config_inputdata.xml

@@ -6,9 +6,9 @@ derivative_mod.o \
 bndry_mod.o \
 prim_advance_mod.o
 
-# CLM's SatellitePhenologyMod is compiled incorrectly with intel 15.0.0 at -O2
+# 
 REDUCED_OPT_OBJS=\
-SatellitePhenologyMod.o
+mo_rxt_rates_conv.o
 
 # shr_wv_sat_mod does not need to have better than ~0.1% precision, and benefits
 # enormously from a lower precision in the vector functions.

machines/Depends.gnu

@@ -6,9 +6,15 @@ derivative_mod.o \
 bndry_mod.o \
 prim_advance_mod.o
 
-# CLM's SatellitePhenologyMod is compiled incorrectly with intel 15.0.0 at -O2
+# FMS netcdf_io gives ICE in intel-oneapi/2024.0.2
+
 REDUCED_OPT_OBJS=\
-SatellitePhenologyMod.o 
+netcdf_io.o \
+fms_netcdf_domain_io.o \
+fms_netcdf_unstructured_domain_io.o
+
+REDUCED_DEBUG_OBJS=\
+unicon.o
 
 # shr_wv_sat_mod does not need to have better than ~0.1% precision, and benefits
 # enormously from a lower precision in the vector functions.
@@ -38,7 +44,7 @@ ifeq ($(DEBUG),FALSE)
   $(PERFOBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O3  -no-prec-div $<
   $(REDUCED_OPT_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O1 $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS)  -O0 $<
   $(REDUCED_PRECISION_OBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -fimf-precision=low -fp-model fast $<
   $(SHR_RANDNUM_FORT_OBJS): %.o: %.F90
@@ -47,5 +53,8 @@ ifeq ($(DEBUG),FALSE)
 	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fp-model fast $<
   $(PUMAS_MG_OBJS): %.o: %.F90
 	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -no-fma -ftz -no-prec-sqrt -qoverride-limits -no-inline-max-total-size -inline-factor=200 -qopt-report=5 $<
+else
+  $(REDUCED_DEBUG_OBJS): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O0 -check nobounds -check nopointers $<
 
 endif

machines/Depends.intel

@@ -13,6 +13,20 @@ REDUCED_OPT_OBJS=\
 SatellitePhenologyMod.o \
 esm.o
 
+REDUCED_ERRORCHKS_IN_DEBUG=\
+fvm_consistent_se_cslam.o \
+clm_instMod.o \
+mpp.o \
+horiz_interp_conserve.o \
+data_override.o \
+MOM_io_infra.o \
+dynColumnStateUpdaterMod.o \
+prim_advection_mod.o \
+MARBL_forcing_mod.o \
+MOM_internal_tide_input.o \
+MOM_ice_shelf.o \
+mom_surface_forcing_nuopc.o
+
 REDUCED_PRECISION_OBJS=\
 shr_wv_sat_mod.o
 
@@ -61,6 +75,7 @@ mo_rte_solver_kernels.o \
 mo_optical_props_kernels.o
 
 CLUBB_OBJS=\
+clubb_intr.o\
 adg1_adg2_3d_luhar_pdf.o\
 advance_clubb_core_module.o\
 advance_helper_module.o\
@@ -154,9 +169,18 @@ ifeq ($(DEBUG),FALSE)
 #  $(SHR_RANDNUM_C_OBJS): %.o: %.c
 #	  $(CC) -c $(INCLDIR) $(INCS) $(CFLAGS) -O3 -fast $<
   $(PUMAS_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(GPUFLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -O3 -fastsse -Mnofma -Mflushz -Mfprelaxed=sqrt $(OPENACC_GPU_FLAGS) $<
   $(RRTMGP_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
   $(CLUBB_OBJS): %.o: %.F90
-	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(GPUFLAGS) $<
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) $(OPENACC_GPU_FLAGS) $<
+
+ifeq ($(COMP_NAME),mpaso)
+  # mpas ocean files need gpuflags
+  FFLAGS +=$(GPUFLAGS)
+endif
+
+else
+  $(REDUCED_ERRORCHKS_IN_DEBUG): %.o: %.F90
+	  $(FC) -c $(INCLDIR) $(INCS) $(FFLAGS) $(FREEFLAGS) -Mnobounds -Ktrap=none -Knoieee $<
 endif

machines/Depends.intel-oneapi

@@ -0,0 +1,14 @@
+if (COMP_NAME STREQUAL gptl)
+  string(APPEND CPPDEFS " -DHAVE_NANOTIME -DBIT64 -DHAVE_VPRINTF -DHAVE_BACKTRACE -DHAVE_SLASHPROC -DHAVE_COMM_F2C -DHAVE_TIMES -DHAVE_GETTIMEOFDAY")
+endif()
+set(MPI_SERIAL_PATH "$ENV{NCAR_ROOT_MPI_SERIAL}")
+set(NETCDF_PATH "$ENV{NETCDF}")
+set(PIO_FILESYSTEM_HINTS "lustre")
+set(PNETCDF_PATH "$ENV{PNETCDF}")
+if(DEFINED ENV{PIO})
+  set(PIO_LIBDIR "$ENV{PIO}/lib")
+  set(PIO_INCDIR "$ENV{PIO}/include")
+endif()
+# If we want to use cray-libsci instead of mkl uncomment this line as well as the module in config_machines.xml
+string(REPLACE "-mkl=cluster" "" SLIBS "${SLIBS}")
+string(APPEND CPPDEFS " -DHAVE_GETTID")

machines/Depends.nvhpc

@@ -4,13 +4,13 @@
   <submit_args>
     <argument> -l gpu_type=$GPU_TYPE </argument>
   </submit_args>
-  <directives queue="casper" compiler="nvhpc" gpu_enabled="true">
+  <directives queue="casper" gpu_enabled="true">
     <directive default="/bin/bash" > -S {{ shell }} </directive>
-    <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=700GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
+    <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem={{ mem_per_node }}GB:ngpus={{ ngpus_per_node }}:mps=1 </directive>
   </directives>
   <directives queue="casper" gpu_enabled="false">
     <directive default="/bin/bash" > -S {{ shell }} </directive>
-    <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem=300GB:ngpus={{ ngpus_per_node }} </directive>
+    <directive> -l select={{ num_nodes }}:ncpus={{ max_tasks_per_node }}:mpiprocs={{ tasks_per_node }}:ompthreads={{ thread_count }}:mem={{ mem_per_node }}GB:ngpus={{ ngpus_per_node }} </directive>
   </directives>
   <!-- Unknown queues use the batch directives for the regular queue -->
   <unknown_queue_directives>casper</unknown_queue_directives>