Merge pull request #345 from CFD-GO/develop

Version 6.5.1

Author: llaniewski

.travis.yml

@@ -45,12 +45,14 @@ env:
     - MODEL=d3q27 
     - MODEL=d3q27_cumulant 
     - MODEL=d3q27_cumulant_AVG_IB_SMAG
+    - MODEL=d2q9_adj
 
 before_install:
     - tools/install.sh rdep
     - tools/install.sh rinside
     - sudo tools/install.sh python-dev
     - tools/install.sh submodules
+    - "[[ ! $MODEL =~ _adj$ ]] || tools/install.sh tapenade"
 
 install:
     - nvcc --version

src/Handlers/cbStop.cpp.Rt

@@ -22,6 +22,8 @@ int cbStop::Init () {
 		        for (g in rows(Globals)) { ?>
 		attr = node.attribute("<?%s g$name ?>Change");
 		if (attr) AddStop(<?%s g$Index ?>, STOP_CHANGE, attr.as_double());
+		attr = node.attribute("<?%s g$name ?>PercentChange");
+		if (attr) AddStop(<?%s g$Index ?>, STOP_PERCENTCHANGE, attr.as_double());
 		attr = node.attribute("<?%s g$name ?>Above");
 		if (attr) AddStop(<?%s g$Index ?>, STOP_ABOVE, attr.as_double());
 		attr = node.attribute("<?%s g$name ?>Below");
@@ -63,6 +65,10 @@ int cbStop::DoIt () {
 					if (fabs(old[i] - v) > limit[i]) any++;
 					if (D_MPI_RANK == 0) output("    change:      %4lg < %4lg", fabs(old[i] - v), limit[i]);
 					break;
+				case STOP_PERCENTCHANGE:
+					if (fabs((old[i] - v)/v) > limit[i]) any++;
+					if (D_MPI_RANK == 0) output("    change:    %4lg%% < %4lg%%", 100.0 * fabs((old[i] - v)/v), 100.0 * limit[i]);
+					break;
 				case STOP_ABOVE:
 					if (v < limit[i]) any++;
 					if (D_MPI_RANK == 0) output("    limit:       %4lg > %4lg", v, limit[i]);

src/Handlers/cbStop.h

@@ -9,6 +9,7 @@
 #define STOP_CHANGE 0x01
 #define STOP_ABOVE 0x02
 #define STOP_BELOW 0x03
+#define STOP_PERCENTCHANGE 0x04
 
 class  cbStop  : public  Callback  {
         std::vector< int > what;

src/LatticeContainer.inc.cpp.Rt

@@ -268,7 +268,7 @@ template <class N> CudaGlobalFunction void RunKernel()
 	now.x_ = CudaThread.x + CudaBlock.z*CudaNumberOfThreads.x + constContainer.dx;
 	now.y_ = CudaThread.y + CudaBlock.x*CudaNumberOfThreads.y + constContainer.dy;
 	now.z_ = CudaBlock.y+constContainer.dz;
-	#ifndef GRID_3D
+	#ifndef GRID3D
 		for (; now.x_ < constContainer.nx; now.x_ += CudaNumberOfThreads.x) {
 	#endif
 		now.Pre();
@@ -278,7 +278,7 @@ template <class N> CudaGlobalFunction void RunKernel()
 			now.OutOfDomain();
 		}
 		now.Glob();
-	#ifndef GRID_3D
+	#ifndef GRID3D
 		}
 	#endif
 }
@@ -445,7 +445,7 @@ template <class N> inline void LatticeContainer::RunInteriorT(CudaStream_t inter
     int nnz;
     ky = ThreadNumber< N >::getKY();
     nnz = fz - dz;
-    #ifdef GRID_3D
+    #ifdef GRID3D
         CudaKernelRunNoWait( RunKernel<N> , dim3(ky, nnz, kx) , dim3(X_BLOCK,ThreadNumber< N >::getSY()),(),interiorStream);
     #else
         CudaKernelRunNoWait( RunKernel<N> , dim3(ky, nnz, 1) , dim3(X_BLOCK,ThreadNumber< N >::getSY()),(),interiorStream);

src/Makefile.in.Rt

@@ -25,7 +25,7 @@ LIBS := @R_LIB@ @LIBS@ @RCPP_LD@ -Xlinker -rpath -Xlinker $(MPI_LIBS) -L$(MPI_LI
 
 TARGETS := CLB.so
 
-NVCC := @CUDA_HOME@/bin/nvcc -gencode arch=compute_10,code=sm_10 -gencode arch=compute_11,code=sm_11 -gencode arch=compute_12,code=sm_12 -gencode arch=compute_13,code=sm_13 -gencode arch=compute_20,code=sm_20  -gencode arch=compute_60,code=sm_60
+NVCC := @CUDA_HOME@/bin/nvcc -gencode arch=compute_10,code=sm_10 -gencode arch=compute_11,code=sm_11 -gencode arch=compute_12,code=sm_12 -gencode arch=compute_13,code=sm_13 -gencode arch=compute_20,code=sm_20  -gencode arch=compute_60,code=sm_60, -gencode arch=compute_75,code=sm_75
 
 all: $(TARGETS) 
 

src/config.main.mk.in

@@ -9,6 +9,7 @@ TOOLS = tools
 RT = $(TOOLS)/RT -b
 MKPATH = $(TOOLS)/mkpath
 RS = R --slave --quiet --vanilla -f
+TAPENADE = @TAPENADE@
 ADMOD = tools/ADmod.R
 
 RTOPT=@RTOPT@

src/configure.ac

@@ -51,7 +51,7 @@ AC_ARG_ENABLE([cpp11],
 
 AC_ARG_WITH([cuda-arch],
 	AC_HELP_STRING([--with-cuda-arch=arch],
-		[specify the desired CUDA architecture (sm_11/sm_13/sm_20/sm_30/sm_60)]))
+		[specify the desired CUDA architecture (sm_11/sm_13/sm_20/sm_30/sm_60/sm_75)]))
 
 AC_ARG_WITH([nlopt],
 	AC_HELP_STRING([--with-nlopt=nlopt],
@@ -76,6 +76,10 @@ AC_ARG_WITH([lammps],
 AC_ARG_WITH([lammps-lib],
 	AC_HELP_STRING([--with-lammps-lib=libfile],
 		[specify the name of the lammps/liggghts library (.so/.a)]))
+
+AC_ARG_WITH([tapenade],
+	AC_HELP_STRING([--with-tapenade=path],
+		[use tapenade for code differentiation for adjoint]))
 
 
 AC_ARG_WITH([verbosity_level],
@@ -281,7 +285,14 @@ then
 		AC_MSG_RESULT([${CUDA_ARCH}])
 		AC_DEFINE([GRID3D], [1], [Using 3D block grid in CUDA])
 
-    CUDA_SUFF="60"
+    	CUDA_SUFF="60"
+	
+	elif test "$CUDA_ARCH" == "sm_75"
+	then
+		AC_MSG_RESULT([${CUDA_ARCH}])
+		AC_DEFINE([GRID3D], [1], [Using 3D block grid in CUDA])
+
+    	CUDA_SUFF="75"
 	else
 		AC_MSG_ERROR([unknown arch ${CUDA_ARCH}])
 	fi
@@ -655,6 +666,44 @@ then
 	fi
 fi
 
+if test "x${with_tapenade}" != "xno"
+then
+	if test "x${with_tapenade}" == "xyes" || test "x${with_tapenade}" == "x"
+	then
+		if test -f "tapenade/bin/tapenade"
+		then
+			TAPENADE="$PWD/tapenade/bin/tapenade"
+		else
+			TAPENADE="tapenade"
+		fi
+	else
+		if test -d "${with_tapenade}"
+		then
+			TAPENADE="$(cd ${with_tapenade}; pwd)/bin/tapenade"
+		else
+			AC_MSG_ERROR([Path '${with_tapenade}' is not a directory])
+		fi
+	fi
+	
+	
+	AC_MSG_CHECKING([for tapenade])
+	if command -v "${TAPENADE}" >/dev/null 2>&1
+	then
+		AC_MSG_RESULT([yes])
+		TAPENADE="$(command -v "${TAPENADE}")"
+	else
+		AC_MSG_RESULT([no])
+		TAPENADE=""
+		if test "x${with_tapenade}" != "x"
+		then
+			AC_MSG_ERROR([Tapenade '$TAPENADE' is not a valid executable])
+		fi
+	fi
+fi
+		
+		
+
+
 if test "x${enable_debug}" == "xyes"
 then
 	if test "x${with_verbosity_level}" == "x"
@@ -779,6 +828,7 @@ AC_SUBST(CROSS_CPU)
 AC_SUBST(WITH_CATALYST)
 AC_SUBST(WITH_LAMMPS)
 AC_SUBST(WITH_R)
+AC_SUBST(TAPENADE)
 AC_SUBST(PV_VERSION)
 AC_SUBST(RTOPT)
 AC_SUBST(CPP_OPT)

src/makefile.main.Rt

@@ -211,7 +211,7 @@ wiki/schema/<?%s m ?>.xsd:$(SRC)/schema.xsd.Rt $(SRC)/conf.R doc/elements.yaml <
 
 <?%s d ?>/<?%s m ?>/tapenade.run : tools/makeAD <?%s d ?>/<?%s m ?>/Dynamics.c <?%s d ?>/<?%s m ?>/ADpre.h <?%s d ?>/<?%s m ?>/ADpre_b.h <?%s d ?>/<?%s m ?>/Dynamics.h <?%s d ?>/<?%s m ?>/Consts.h <?%s d ?>/<?%s m ?>/types.h <?%s d ?>/<?%s m ?>/ADset.sh <?%s d ?>/<?%s m ?>/ADpost.sed
 	@echo "  TAPENADE   $<"
-	@(cd <?%s d ?>/<?%s m ?>; ../../tools/makeAD)
+	@(cd <?%s d ?>/<?%s m ?>; ../../tools/makeAD "$(TAPENADE)")
 
 <?%s d ?>/<?%s m ?>/dep.mk:tools/dep.R <?%s src ?>
 	@echo "  AUTO-DEP   $@"

tools/install.sh

@@ -393,6 +393,29 @@ do
 		try "Leaving module directory" cd ..
 		try "Remember to restart terminal" . ~/.bashrc	
 		;;
+	tapenade)
+		if echo "$2" | grep -Eq '^[0-9]*[.][0-9]*$'
+		then
+			shift
+			VER="$1"
+		else
+			VER="3.16"
+		fi
+		if test -d ../tapenade
+		then
+			echo "Looks like tapenade already is installed at '$(cd ../tapenadel; pwd))'"
+			exit -1
+		fi
+		try "Downloading Tapenade ($VER)" wget $WGETOPT http://www-sop.inria.fr/ecuador/tapenade/distrib/tapenade_$VER.tar
+		try "Unpacking Tapenade" tar xf tapenade_$VER.tar
+		if test -d tapenade_$VER
+		then
+			mv tapenade_$VER ../tapenade
+			echo "Installed Tapenade at '$(cd ../tapenade; pwd)'"
+		else
+			echo "Tapenade installation failed"
+		fi
+		;;
 	-*)
 		echo "Unknown option $1" ; usage ;;
 	*)		

tools/makeAD

@@ -2,6 +2,19 @@
 
 path_to_script=$(dirname $0)
 #cd package/src
+
+if test -z "$1"
+then
+	TAPENADE="tapenade"
+else
+	TAPENADE="$1"
+fi
+if ! command -v "$TAPENADE"
+then
+	echo "Tapenade not found ('$TAPENADE')" >&2
+	exit -1
+fi
+
 . ADset.sh
 rm Dynamics_tmp.c Dynamics_b.c* ADpre_.h types_b.h tmp 2>/dev/null
 echo "#define CudaDeviceFunction" > Dynamics_tmp.c
@@ -16,11 +29,11 @@ cat Dynamics.c | sed 's/pow(\([^,]*\),3)/((\1)*(\1)*(\1))/g' >> Dynamics_tmp.c
 echo Variables for differentiation: $VARIABLES
 echo Additional variables for differentiation: $SETTINGS
 echo tapenade -html -msglevel 12 -b Dynamics_tmp.c -root $MAIN_FUN -o DynamicsS -vars "$VARIABLES $SETTINGS" -outvars "$VARIABLES $SETTINGS"
-tapenade -msglevel 12 -b Dynamics_tmp.c -root $MAIN_FUN -o DynamicsS -vars "$VARIABLES $SETTINGS" -outvars "$VARIABLES $SETTINGS"
+$TAPENADE -msglevel 12 -b Dynamics_tmp.c -root $MAIN_FUN -o DynamicsS -vars "$VARIABLES $SETTINGS" -outvars "$VARIABLES $SETTINGS"
 echo "#define CONST_SETTINGS" > ADpre_.h
 cat ADpre.h >> ADpre_.h
 echo tapenade -html -msglevel 12 -b Dynamics_tmp.c -root $MAIN_FUN -o DynamicsS -vars "$VARIABLES $SETTINGS" -outvars "$VARIABLES $SETTINGS"
-tapenade -msglevel 12 -b Dynamics_tmp.c -root $MAIN_FUN -o Dynamics -vars "$VARIABLES" -outvars "$VARIABLES"
+$TAPENADE -msglevel 12 -b Dynamics_tmp.c -root $MAIN_FUN -o Dynamics -vars "$VARIABLES" -outvars "$VARIABLES"
 for f in Dynamics_b.c DynamicsS_b.c
 do
 	test -f $f || { echo "Tapenade failed"; exit -1; }