Update AnsysMechanical.sbatch

Ansys Mechanical gets the num of cores from the hostlist

Author: nicolaven

apps/AnsysMechanical/AnsysMechanical.sbatch

@@ -1,7 +1,7 @@
 #!/bin/bash
 #SBATCH --exclusive
-#SBATCH --nodes=1
-#SBATCH --ntasks=64
+#SBATCH --nodes=2
+#SBATCH --ntasks=128
 #SBATCH --constraint=hpc6id.32xlarge
 #SBATCH --partition=hpc6id
 #SBATCH --chdir=/fsx/slurm-out/mechanical
@@ -15,33 +15,33 @@ export I_MPI_FABRICS=shm:ofi
 export I_MPI_OFI_PROVIDER=efa
 export I_MPI_MULTIRAIL=1
 #export FI_EFA_RECVWIN_SIZE=65536
+export FI_EFA_FORK_SAFE=1
 module load libfabric-aws
 export INTELMPI_ROOT="$(dirname $(dirname $(which mpirun)))"
 ######################## EFA settings ######################## 
 
-
 export ANSYSLMD_LICENSE_FILE=${ANSYSLMD_LICENSE_FILE:-"[email protected]"}
 
-mechanicalversion=${1:-"231"}
+mechanicalversion=${1:-"v251"}
 input_file=${2:-"/fsx/.../your_file.dat"}
 basedir=${BASE_DIR:-"/fsx"}
 
-APP_BIN_PATH="${basedir}/ansys_inc/v{${mechanicalversion}/ansys/bin/ansys${mechanicalversion}"
-
+APP_BIN_PATH="${basedir}/ansys_inc/${mechanicalversion}/ansys/bin/mapdl"
 TOKEN=$(curl -s -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 21600")
 workdir="$(readlink -m "${basedir}/${SLURM_JOB_NAME%.*}")/Run/${SLURM_JOB_ID}-${SLURM_JOB_NUM_NODES}x$(curl -s -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/instance-type)-${SLURM_NPROCS}-$(date "+%d-%m-%Y-%H-%M")"
 
 mkdir -p "${workdir}" && cd "${workdir}"
 
-scontrol show nodes $SLURM_NODELIST | grep NodeHostName= | awk '{print $2}' | sed 's/NodeHostName=//' >hostfile
 export cores_x_node=$(((SLURM_NPROCS / SLURM_JOB_NUM_NODES ) + ( SLURM_NPROCS % SLURM_JOB_NUM_NODES > 0 )))
 
-MACHINES=$(xargs printf '%s:$cores_x_node, ' < hostfile)
+for i in $(scontrol show hostnames=$SLURM_JOB_NODELIST); do
+  machines=$machines:$i:$cores_x_node
+done
+machines=${machines:1}
 
 cp $0 .
 ln -s "${input_file}" .
 
-
 echo "Drop caches on all nodes"
 mpirun -np $SLURM_JOB_NUM_NODES -ppn 1 /bin/bash -c "sync && echo 3 | sudo tee /proc/sys/vm/drop_caches"
 
@@ -51,5 +51,5 @@ mpirun -np $SLURM_JOB_NUM_NODES -ppn 1 /bin/bash -c "echo always | sudo tee /sys
 echo "Install missing lib"
 mpirun -np $SLURM_JOB_NUM_NODES -ppn 1 /bin/bash -c "sudo yum -y install mesa-libGLU"
 
-
-"${APP_BIN_PATH}" -b -dis -mpi intelmpi -ssh -np ${SLURM_NPROCS} -machines $MACHINES -i $(basename ${input_file}) -o output.log
+# Ansys Mechanical gets the total number of cores from hostlist not from -np
+"${APP_BIN_PATH}" -b -dis -mpi intelmpi -ssh -machines $MACHINES -i $(basename ${input_file}) -o output.log