Skip to content
CI PARALLEL (MPI) GPU

CI PARALLEL (MPI) GPU #54

Sign in to view logs

CI PARALLEL (MPI) GPU

CI PARALLEL (MPI) GPU #54

Workflow file for this run

.github/workflows/CI_parallel_GPU.yml at 1970abb
name: CI PARALLEL (MPI) GPU
########################################################################
on:
# Manual trigger only
workflow_dispatch:
jobs:
wait-guard:
name: Runner availability guard
runs-on: ubuntu-latest
timeout-minutes: 2880 # 48 hours
steps:
# This job serves as a guard to prevent the workflow from pending indefinitely if the self-hosted GPU runner is not available.
- run: echo "GPU runner did not appear within 48 hours"
parallel-gpu-tests:
# Uses self-hosted runner on nibbler cluster with GPU access
name: Parallel GPU Tests (NVFORTRAN)
runs-on: [self-hosted, nibbler, gpu]
# Prevent runaway jobs from consuming cluster resources indefinitely
timeout-minutes: 120
env:
METIS_HOME: ${{ github.workspace }}/metis-5.1.0/build/Linux-x86_64
steps:
- uses: actions/checkout@v4
# Fix ownership so the SLURM job user can read all checked-out files.
# Root (or the runner service account) owns files after checkout; a.ballout
# needs read access to mesh files, binaries, and control files at runtime.
- name: Fix workspace ownership
run: sudo chown -R a.ballout:a.ballout $GITHUB_WORKSPACE
# Install METIS library (required for parallel partitioning)
- name: Install METIS
run: |
module purge
module load cmake
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
if [ ! -d "metis-5.1.0" ]; then
wget "https://src.fedoraproject.org/lookaside/pkgs/metis/metis-5.1.0.tar.gz/5465e67079419a69e0116de24fce58fe/metis-5.1.0.tar.gz"
tar -xvf metis-5.1.0.tar.gz
cd metis-5.1.0
# Update CMake minimum required version to 3.5
sed -i 's/cmake_minimum_required(VERSION [0-9.]\+)/cmake_minimum_required(VERSION 3.5)/' CMakeLists.txt
make config cc=gcc
make
mkdir -p build/Linux-x86_64/lib build/Linux-x86_64/bin
cp -r build/Linux-x86_64/libmetis/* build/Linux-x86_64/lib/ 2>/dev/null || true
cp -r build/Linux-x86_64/programs/* build/Linux-x86_64/bin/ 2>/dev/null || true
fi
- name: Configure Horses3D
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cd Solver
./configure
- name: Compile horses3d
working-directory: ./Solver
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make allclean COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
make ns ins mu COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
########################################################################
####### NAVIER-STOKES TESTS ########
########################################################################
#
# 1) CYLINDER
# -----------
- name: Build NSCylinder
working-directory: ./Solver/test/NavierStokes/Cylinder/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Fix test directory ownership
run: sudo chown -R a.ballout:a.ballout $GITHUB_WORKSPACE
if: '!cancelled()'
- name: Run NSCylinder
working-directory: ./Solver/test/NavierStokes/Cylinder
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinder
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2 # number of gpus max 4
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns Cylinder.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
# Check exit code (ExitCode:Signal, e.g., "0:0" means success)
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 5) CYLINDER SMAGORINSKY
# -----------------------
- name: Build NSCylinderSmagorinsky
working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderSmagorinsky
working-directory: ./Solver/test/NavierStokes/CylinderSmagorinsky
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderSmagorinsky
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2 # number of gpus max 4
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderSmagorinsky.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
# Check exit code (ExitCode:Signal, e.g., "0:0" means success)
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 6) CYLINDER WALE
# ------------------
- name: Build NSCylinderWALE
working-directory: ./Solver/test/NavierStokes/CylinderWALE/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderWALE
working-directory: ./Solver/test/NavierStokes/CylinderWALE
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderWALE
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2 # number of gpus max 4
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderWALE.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 7) CYLINDER VREMAN
# ------------------
- name: Build NSCylinderVreman
working-directory: ./Solver/test/NavierStokes/CylinderVreman/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderVreman
working-directory: ./Solver/test/NavierStokes/CylinderVreman
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderVreman
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2 # number of gpus max 4
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderVreman.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 8) CYLINDER ChandrasekarRoe
# ------------------
- name: Build NSCylinderChandrasekarRoe
working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderChandrasekarRoe
working-directory: ./Solver/test/NavierStokes/CylinderChandrasekarRoe
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderChandrasekarRoe
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2 # number of gpus max 4
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderChandrasekarRoe.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 9) TAYLOR GREEN VORTEX
# ----------------------
- name: Build TaylorGreen
working-directory: ./Solver/test/NavierStokes/TaylorGreen/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run TaylorGreen
working-directory: ./Solver/test/NavierStokes/TaylorGreen
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=TaylorGreen
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2 # number of gpus max 4
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns TaylorGreen.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
# Check exit code (ExitCode:Signal, e.g., "0:0" means success)
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 18) NACA0012 Unsteady Dual Time Stepping
# ----------------------------
- name: Build DualTimeStepping
working-directory: ./Solver/test/NavierStokes/DualTimeStepping/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run DualTimeStepping
working-directory: ./Solver/test/NavierStokes/DualTimeStepping
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=DualTimeStepping
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2 # number of gpus max 4
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns DualTimeStepping.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 26) IBM CYLINDER
# -------------------
- name: Build IBM_Cylinder
working-directory: ./Solver/test/NavierStokes/IBM_Cylinder/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run IBM_Cylinder
working-directory: ./Solver/test/NavierStokes/IBM_Cylinder
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=IBM_Cylinder
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2 # number of gpus max 4
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns IBMCylinder.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 40) CYLINDER EntropyConservingCentral
# -------------------------------------
- name: Build NSCylinderEntropyConservingCentral
working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderEntropyConservingCentral
working-directory: ./Solver/test/NavierStokes/CylinderEntropyConservingCentral
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderEntropyConservingCentral
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderEntropyConservingCentral.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 41) CYLINDER RoePikePirozzoli
# ----------------------------
- name: Build NSCylinderRoePikePirozzoli
working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderRoePikePirozzoli
working-directory: ./Solver/test/NavierStokes/CylinderRoePikePirozzoli
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderRoePikePirozzoli
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderRoePikePirozzoli.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 42) CYLINDER KennedyGruberLaxFriedrichs
# ---------------------------------------
- name: Build NSCylinderKennedyGruberLaxFriedrichs
working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderKennedyGruberLaxFriedrichs
working-directory: ./Solver/test/NavierStokes/CylinderKennedyGruberLaxFriedrichs
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderKennedyGruberLaxFriedrichs
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderKennedyGruberLaxFriedrichs.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 43) CYLINDER MorinishiLowDissipationRoe
# ----------------------------------------
- name: Build NSCylinderMorinishiLowDissipationRoe
working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderMorinishiLowDissipationRoe
working-directory: ./Solver/test/NavierStokes/CylinderMorinishiLowDissipationRoe
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderMorinishiLowDissipationRoe
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderMorinishiLowDissipationRoe.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 44) CYLINDER DucrosMatrixDissipation
# -------------------------------------
- name: Build NSCylinderDucrosMatrixDissipation
working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderDucrosMatrixDissipation
working-directory: ./Solver/test/NavierStokes/CylinderDucrosMatrixDissipation
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderDucrosMatrixDissipation
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderDucrosMatrixDissipation.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 45) CYLINDER ViscousStandard
# -------------------------
- name: Build NSCylinderViscousStandard
working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderViscousStandard
working-directory: ./Solver/test/NavierStokes/CylinderViscousStandard
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderViscousStandard
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderViscousStandard.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 46) CYLINDER RusanovStandard
# -------------------------
- name: Build NSCylinderRusanovStandard
working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderRusanovStandard
working-directory: ./Solver/test/NavierStokes/CylinderRusanovStandard
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderRusanovStandard
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderRusanovStandard.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 47) CYLINDER UdissStandard
# -------------------------
- name: Build NSCylinderUdissStandard
working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderUdissStandard
working-directory: ./Solver/test/NavierStokes/CylinderUdissStandard
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderUdissStandard
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns CylinderUdissStandard.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 48) CYLINDER sprk33
# -------------------
- name: Build NSCylinderssprk33
working-directory: ./Solver/test/NavierStokes/Cylinderssprk33/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderssprk33
working-directory: ./Solver/test/NavierStokes/Cylinderssprk33
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderssprk33
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns Cylinderssprk33.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 49) CYLINDER sprk43
# -------------------
- name: Build NSCylinderssprk43
working-directory: ./Solver/test/NavierStokes/Cylinderssprk43/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSCylinderssprk43
working-directory: ./Solver/test/NavierStokes/Cylinderssprk43
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSCylinderssprk43
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns Cylinderssprk43.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
#
# 50) Limiter Test
# -------------------
- name: Build NSLimiterTest
working-directory: ./Solver/test/NavierStokes/LimiterTest/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make ns COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run NSLimiterTest
working-directory: ./Solver/test/NavierStokes/LimiterTest
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=NSLimiterTest
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.ns LimiterTest.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'
########################################################################
####### INCOMPRESSIBLE_NS TESTS ########
########################################################################
### Sometimes this test passes and sometimes it failscc
#
# 1) NACA0012
# ----------------------------
# - name: Build iNS-NACA0012
# working-directory: ./Solver/test/IncompressibleNS/NACA0012/SETUP
# run: |
# module purge
# module load hpc_sdk
# module load nvhpc/24.11
# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
# export CUDA_HOME=$NVHPC_CUDA_HOME
# export CUDA_PATH=$NVHPC_CUDA_HOME
# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
# make clean || true
# make ins COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
# if: '!cancelled()'
# - name: Run iNS-NACA0012
# working-directory: ./Solver/test/IncompressibleNS/NACA0012
# run: |
# module purge
# module load hpc_sdk
# module load nvhpc/24.11
# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
# export CUDA_HOME=$NVHPC_CUDA_HOME
# export CUDA_PATH=$NVHPC_CUDA_HOME
# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
# cat > run_test.sh << 'EOF'
# #!/bin/bash
# #SBATCH --job-name=NACA0012
# #SBATCH --ntasks=2
# #SBATCH --partition=gpu
# #SBATCH --cpus-per-task=1
# #SBATCH --mem=32G
# #SBATCH --gres=gpu:2 # number of gpus max 4
# #SBATCH --time=00:15:00
# module purge
# module load hpc_sdk
# module load nvhpc/24.11
# export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
# export CUDA_HOME=$NVHPC_CUDA_HOME
# export CUDA_PATH=$NVHPC_CUDA_HOME
# export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
# export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
# export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
# export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
# mpirun -np 2 --bind-to none ./horses3d.ins NACA0012.control
# EOF
# job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
# while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
# cat slurm-${job_id}.out
# exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
# if [ "$exit_code" != "0" ]; then
# echo "Test failed with exit code: $exit_code"
# exit 1
# fi
# if: '!cancelled()'
########################################################################
####### MULTIPHASE TESTS ########
########################################################################
#
# 4) Entropy conserving test
# ----------------------------
- name: Build MultiphaseEntropyConservingTest
working-directory: ./Solver/test/Multiphase/EntropyConservingTest/SETUP
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
make clean || true
make mu COMPILER=nvfortran COMM=PARALLEL ENABLE_THREADS=NO WITH_METIS=YES
if: '!cancelled()'
- name: Run MultiphaseEntropyConservingTest
working-directory: ./Solver/test/Multiphase/EntropyConservingTest
run: |
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
cat > run_test.sh << 'EOF'
#!/bin/bash
#SBATCH --job-name=EntropyConservingTest
#SBATCH --ntasks=2
#SBATCH --partition=gpu
#SBATCH --mem=32G
#SBATCH --gres=gpu:2 # number of gpus max 4
#SBATCH --time=00:15:00
module purge
module load hpc_sdk
module load nvhpc/24.11
export NVHPC_CUDA_HOME=$NVHPC_ROOT/cuda
export CUDA_HOME=$NVHPC_CUDA_HOME
export CUDA_PATH=$NVHPC_CUDA_HOME
export LD_LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LD_LIBRARY_PATH
export LIBRARY_PATH=$NVHPC_ROOT/math_libs/12.6/targets/x86_64-linux/lib:$NVHPC_CUDA_HOME/lib64:$LIBRARY_PATH
export NVCOMPILER_COMM_LIBS_HOME=$NVHPC_ROOT/comm_libs/12.6
export OPAL_PREFIX=$NVHPC_ROOT/comm_libs/mpi
mpirun -np 2 --bind-to none ./horses3d.mu EntropyConservingTest.control
EOF
job_id=$(sbatch --uid=a.ballout --parsable run_test.sh)
while squeue -j $job_id 2>/dev/null | grep -q $job_id; do sleep 10; done
cat slurm-${job_id}.out
exit_code=$(sacct -j $job_id --format=ExitCode --noheader | head -1 | tr -d ' ' | cut -d':' -f1)
if [ "$exit_code" != "0" ]; then
echo "Test failed with exit code: $exit_code"
exit 1
fi
if: '!cancelled()'