Merge pull request #644 from rjleveque/bouss_petsc_updates

Updates to bouss files for PETSc setup

Author: rjleveque

examples/bouss/README.txt

@@ -1,5 +1,8 @@
 
-GeoClaw Boussinesq solver examples
+GeoClaw Boussinesq solver examples (in 2D)
+
+Some 1D Boussinesq solver examples are in $CLAW/geoclaw/examples/1d_classic
+in directories starting with bouss_
 
 The radial_flat subdirectory contains one example using the Boussinesq
 solver introduced in Clawpack v5.10.0.
@@ -13,8 +16,12 @@ which are described in the paper found at:
 
 Running these codes requires PETSc  Version 3.20 (or later) in order to use
 OpenMP along with MPI.  Some flags have to be set as environment variables
-or directly in the application Makefile, e.g. see the lines commented out in
-radial_flat/Makefile.
+or directly in the application Makefile.
+
+For more information on installing PETSc and setting these environment
+variables properly, see:
+  https://www.clawpack.org/bouss2d.html
+
 
 **Update:** Clawpack 5.12.0 now requires PETSc Version 3.23 (or later).
 
@@ -24,6 +31,38 @@ variables for the bash shell.
 A file petscMPIoptions is also required to set some PETSc parameters for the
 iterative method used to solve the large sparse linear systems that arise at 
 each refinement level when the Boussinesq equations are solved. 
-One of the environment variables mentioned above points to this file, and a
-sample is included in this directory.
+The environment variable PETSC_OPTIONS should point to the version of this
+file that you wish to use, see setenv.sh for an example of setting this to
+point to the sample petscMPIoptions file included in this directory.
+
+
+**Check:**
+
+The example Makefile in
+    $CLAW/geoclaw/examples/bouss/radial_flat/Makefile
+contains an option so that you can do 
+    $ make check
+at the command line to see how various options are set and to check that
+they look correct (or help debug if the code does not run).
+This is an enhancement of the 'make check' option added to the common
+Makefile for all Clawpack applications in v5.12.0, and should produce
+something like this:
+
+===================
+CLAW = /Users/rjl/git/clawpack
+OMP_NUM_THREADS = 6
+BOUSS_MPI_PROCS = 6
+PETSC_OPTIONS=-options_file /Users/rjl/git/clawpack/geoclaw/examples/bouss/petscMPIoptions
+PETSC_DIR = /Users/rjl/git/Clones/petsc
+PETSC_ARCH = arch-darwin-c-opt
+CLAW_MPIEXEC = mpiexec
+RUNEXE = mpiexec -n 6
+EXE = /Users/rjl/git/clawpack/geoclaw/examples/bouss/radial_flat/xgeoclaw
+CLAW_MPIFC = mpif90
+FC = mpif90
+FFLAGS = -O2 -fopenmp -DHAVE_PETSC -ffree-line-length-none
+LFLAGS = -L/Users/rjl/git/Clones/petsc/arch-darwin-c-opt/lib -lpetsc -fopenmp
+OUTDIR = _output
+PLOTDIR = _plots
+===================
 

examples/bouss/petscMPIoptions

@@ -25,7 +25,7 @@
 
 # preconditioner:
 -pc_type gamg
-#-pc_type hypre
+#-pc_type hypre  # requires installing hypre
 
 
 # debug options:
@@ -36,6 +36,8 @@
 #-ksp_view
 #-info
 
+# print options at end of run:
+-options_view
+
 # test if any options are not used:
 #-options_left
--options_view

examples/bouss/radial_flat/Makefile

@@ -22,13 +22,13 @@ CLAWMAKE = $(CLAW)/clawutil/src/Makefile.common
 
 CLAW_PKG = geoclaw                  # Clawpack package to use
 
-# These environment variables need to be set properly, usually in your
-# shell, but they could be explicitly set here:
-#PETSC_DIR=/full/path/to/petsc
-#PETSC_ARCH=arch-darwin-c-opt
+# Several environment variables are expected in this Makefile,
+# as set for example in the bash script
+#     $CLAW/geoclaw/examples/bouss/setenv.sh
+# Some of these could instead be set here explicitly.
 
+# But PETSC_OPTIONS must be set as an environment variable!
 # The file petscMPIoptions sets parameters for MPI and the iterative solver:
-# PETSC_OPTIONS must be set as an environment variable!
 # e.g. in bash:
 #   export PETSC_OPTIONS="-options_file $CLAW/geoclaw/examples/bouss/petscMPIoptions"
 
@@ -58,7 +58,6 @@ FC = ${CLAW_MPIFC}
 BOUSS_MPI_PROCS ?= 6
 
 EXE = $(PWD)/xgeoclaw
-#RUNEXE="${PETSC_DIR}/${PETSC_ARCH}/bin/mpiexec -n ${BOUSS_MPI_PROCS}"
 RUNEXE="${CLAW_MPIEXEC} -n ${BOUSS_MPI_PROCS}"
 SETRUN_FILE = setrun.py           # File containing function to make data
 OUTDIR = _output                  # Directory for output
@@ -67,7 +66,7 @@ PLOTDIR = _plots                  # Directory for plots
 
 
 # Some compiler flags below are needed for PETSc
-PETSC_INCLUDE = $(PETSC_DIR)/include $(PETSC_DIR)/$(PETSC_ARCH)/include         
+PETSC_INCLUDE = $(PETSC_DIR)/include $(PETSC_DIR)/$(PETSC_ARCH)/include
 INCLUDE += $(PETSC_INCLUDE)
 PETSC_LFLAGS = $(shell PKG_CONFIG_PATH=$(PETSC_DIR)/$(PETSC_ARCH)/lib/pkgconfig pkg-config --libs-only-L --libs-only-l PETSc)
 
@@ -93,7 +92,7 @@ include $(BOUSSLIB)/Makefile.bouss
 
 # ---------------------------------------
 # package sources specifically to exclude
-# (i.e. if a custom replacement source 
+# (i.e. if a custom replacement source
 #  under a different name is provided)
 # ---------------------------------------
 

examples/bouss/setenv.sh

@@ -3,15 +3,42 @@
 # to run the Bouss version of GeoClaw with MPI and OpenMP.
 # Adjust as needed for your system...
 
-# You also need to set CLAW, FC, and perhaps PYTHONPATH
-
-# For more information, see
+# For more information on installing PETSc and setting these environment
+# variables properly, see $CLAW/geoclaw/examples/bouss/README.txt and
 #   https://www.clawpack.org/bouss2d.html
+
+# You also need to set CLAW, and perhaps PYTHONPATH, see:
 #   https://www.clawpack.org/setenv.html
 
+# export CLAW=full/path/to/clawpack
+echo CLAW is set to $CLAW
+
+# path to PETSc installation:
 export PETSC_DIR=/full/path/to/petsc
+
+# PETSC_ARCH is only needed if PETSc is installed inside the PETSc directory.
 export PETSC_ARCH=arch-darwin-c-opt
+# For PETSc installs by conda or package managers, it should not be set.
+#export PETSC_ARCH=
+
+# PETSC_OPTIONS should point to the options file needed to specify
+# many parameters controlling which solver, preconditioner, etc are used:
 export PETSC_OPTIONS="-options_file $CLAW/geoclaw/examples/bouss/petscMPIoptions"
+
+# number of OpenMP threads to use for explicit AMR (as usual in GeoClaw):
 export OMP_NUM_THREADS=6
-export BOUSS_MPI_PROCS=6  # only used in Clawpack Boussinesq example
 
+# number of MPI processes to use for solving linear systems with PETSc:
+export BOUSS_MPI_PROCS=6
+
+# the mpiexec command to use to run the executable:
+export CLAW_MPIEXEC=mpiexec
+# set CLAW_MPIEXEC to mpiexec only if this command is defined in your shell,
+# e.g. to use some version of MPI was installed outside of PETSc.
+# Or set to the full path to this command, e.g. for the PETSc version:
+#export CLAW_MPIEXEC=$PETSC_DIR/$PETSC_ARCH/bin/mpiexec  # requires PETSC_ARCH
+
+# the proper Fortran compiler to use for MPI code:
+# e.g. mpif90 if that is defined in your shell, or gfortran *might* work.
+# This will over-rule any FC environment variable.
+export CLAW_MPIFC=mpif90