diff --git a/.github/workflows/nvhpc.yml b/.github/workflows/nvhpc.yml index e673053926..e6938bca59 100644 --- a/.github/workflows/nvhpc.yml +++ b/.github/workflows/nvhpc.yml @@ -79,7 +79,7 @@ jobs: run: | rm -fr ${NRN_INSTALL_DIR} source "${VENV_DIR}/bin/activate" - export NRN_CONFIG=(-DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_MPI=ON -DCORENRN_ENABLE_GPU=ON -DCMAKE_CXX_COMPILER=nvc++ -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_DOCS=OFF -DNRN_ENABLE_TESTS=ON -DCMAKE_C_COMPILER=nvc -DCMAKE_CUDA_COMPILER=nvcc -DCMAKE_INSTALL_PREFIX="${NRN_INSTALL_DIR}" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DNMODL_ENABLE_FLEX_BISON_LINES=OFF -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -G Ninja -DCMAKE_BUILD_TYPE=Debug) + export NRN_CONFIG=(-DNRN_ENABLE_CORENEURON=ON -DNRN_ENABLE_MPI=ON -DCORENRN_ENABLE_GPU=ON -DNRN_ENABLE_GPU=ON -DNRN_GPU_BACKEND=OpenACC -DCMAKE_CUDA_ARCHITECTURES=75 -DCMAKE_CXX_COMPILER=nvc++ -DNRN_ENABLE_INTERVIEWS=OFF -DNRN_ENABLE_RX3D=OFF -DNRN_ENABLE_DOCS=OFF -DNRN_ENABLE_TESTS=ON -DCMAKE_C_COMPILER=nvc -DCMAKE_CUDA_COMPILER=nvcc -DCMAKE_INSTALL_PREFIX="${NRN_INSTALL_DIR}" -DCMAKE_C_COMPILER_LAUNCHER=ccache -DCMAKE_CXX_COMPILER_LAUNCHER=ccache -DNMODL_ENABLE_FLEX_BISON_LINES=OFF -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache -G Ninja -DCMAKE_BUILD_TYPE=Debug) cmake -B ${NRN_BUILD_DIR} "${NRN_CONFIG[@]}" - name: "Build NEURON" @@ -102,7 +102,10 @@ jobs: CTEST_PARALLEL_LEVEL: 20 run: | source "${VENV_DIR}/bin/activate" - ctest --output-on-failure --test-dir ${NRN_BUILD_DIR} + # Phase B native GPU pre-merge proof (docs/dev/gpu-testing.rst). + # Full ctest -R gpu (~96 tests) is follow-up for this PR. + ctest --output-on-failure --test-dir ${NRN_BUILD_DIR} \ + -R 'testneuron_gpu|_py_gpu_native|external_ringtest::neuron_gpu' - name: "Install NEURON" run: | diff --git a/CMakeLists.txt b/CMakeLists.txt index 8fc7484313..6fe3f86a38 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -109,6 +109,11 @@ option(NRN_ENABLE_MUSIC "Enable MUSIC support" ${NRN_ENABLE_MUSIC_DEFAULT}) option(NRN_ENABLE_RX3D "Enable rx3d support" ${NRN_ENABLE_RX3D_DEFAULT}) option(NRN_ENABLE_NMODL "Enable NMODL code-generator support" ${NRN_ENABLE_NMODL_DEFAULT}) option(NRN_ENABLE_CORENEURON "Enable CoreNEURON support" ${NRN_ENABLE_CORENEURON_DEFAULT}) +option(NRN_ENABLE_GPU "Enable NEURON native GPU support" ${NRN_ENABLE_GPU_DEFAULT}) +set(NRN_GPU_BACKEND + "${NRN_GPU_BACKEND_DEFAULT}" + CACHE STRING "GPU backend for NEURON native GPU (OpenACC only in Phase A-B)") +set_property(CACHE NRN_GPU_BACKEND PROPERTY STRINGS OpenACC) option(NRN_ENABLE_BACKTRACE "Enable pretty-printed backtraces" ${NRN_ENABLE_BACKTRACE_DEFAULT}) option(NRN_ENABLE_TESTS "Enable unit tests" ${NRN_ENABLE_TESTS_DEFAULT}) option(NRN_ENABLE_MATH_OPT "Enable extra math optimisations (to enable SIMD)" @@ -673,6 +678,24 @@ else() set(NRN_CODEGENERATOR_TARGET nocmodl) endif() +# ============================================================================= +# NEURON native GPU (Phase A: option scaffolding; execution still via CoreNEURON) +# ============================================================================= +if(NRN_ENABLE_GPU) + if(NOT NRN_GPU_BACKEND STREQUAL "OpenACC") + message( + FATAL_ERROR + "NRN_GPU_BACKEND=${NRN_GPU_BACKEND} is not supported. Only OpenACC is supported in Phase A-B." + ) + endif() + if(NOT NRN_ENABLE_CORENEURON) + message( + FATAL_ERROR + "NRN_ENABLE_GPU=ON requires NRN_ENABLE_CORENEURON=ON during the Phase A transition. " + "Native GPU execution without CoreNEURON is not yet implemented.") + endif() +endif() + # ============================================================================= # Enable CoreNEURON support # ============================================================================= @@ -703,6 +726,7 @@ if(NRN_ENABLE_CORENEURON) # CoreNEURON exports this list of flags as a property; turn it into a variable in this scope. If # CoreNEURON is installed externally then this is exported into coreneuron-config.cmake. get_property(CORENRN_LIB_LINK_FLAGS GLOBAL PROPERTY CORENRN_LIB_LINK_FLAGS) + get_property(CORENRN_ACC_COMP_FLAGS GLOBAL PROPERTY CORENRN_ACC_COMP_FLAGS) get_property(CORENRN_NEURON_LINK_FLAGS GLOBAL PROPERTY CORENRN_NEURON_LINK_FLAGS) get_property(CORENRN_ENABLE_SHARED GLOBAL PROPERTY CORENRN_ENABLE_SHARED) @@ -752,6 +776,7 @@ endif() # ============================================================================= add_subdirectory(src/sparse13) add_subdirectory(src/gnu) +add_subdirectory(src/neuron) add_subdirectory(src/nrniv) if(NRN_ENABLE_PYTHON) @@ -802,6 +827,14 @@ endif() if(NRN_ENABLE_CORENEURON) list(APPEND NRN_RUN_FROM_BUILD_DIR_ENV "CORENRNHOME=${PROJECT_BINARY_DIR}") endif() +if(NRN_ENABLE_GPU AND CORENRN_ENABLE_GPU) + if(NOT CUDAToolkit_FOUND) + find_package(CUDAToolkit 9.0 QUIET) + endif() + if(CUDAToolkit_FOUND) + prepend_to_var(LD_LIBRARY_PATH "${CUDAToolkit_LIBRARY_DIR}") + endif() +endif() if(NRN_ENABLE_PYTHON) prepend_to_var(PYTHONPATH "${PROJECT_BINARY_DIR}/lib/python:${PROJECT_SOURCE_DIR}/test/rxd") endif() @@ -949,21 +982,42 @@ add_custom_target( COMMAND ${PROJECT_SOURCE_DIR}/external/coding-conventions/bin/format WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) -# Prepare a shell script to format only files modified with respect to master branch +# Prepare a shell script to format only files modified with respect to master/main branch that still +# exist on disk (filters out deleted files). file( WRITE ${CMAKE_CURRENT_BINARY_DIR}/format-pr.sh - "\ -#!bash\n\ -set -e\n\ -cmd='cd ${PROJECT_SOURCE_DIR} && external/coding-conventions/bin/format `git diff --name-only master`'\n\ -echo $cmd\n\ -cd ${PROJECT_SOURCE_DIR} && external/coding-conventions/bin/format `git diff --name-only master`\n\ + "#!/bin/bash +set -euo pipefail + +# Try 'main' first, then fall back to 'master' +for base in main master; do + if git rev-parse --verify \"\$base\" >/dev/null 2>&1; then + BASE_BRANCH=\"\$base\" + break + fi +done + +if [ -z \"\${BASE_BRANCH:-}\" ]; then + echo \"Error: Neither 'main' nor 'master' branch found.\" >&2 + exit 1 +fi + +echo \"Formatting changes vs '\$BASE_BRANCH' (existing files only)...\" >&2 + +# Get changed files, but only pass files that still exist +git diff --name-only \"\$BASE_BRANCH\" | \\ +while read -r file; do + if [ -e \"\$file\" ]; then + printf '%s\\n' \"\$file\" + fi +done | \\ +xargs --no-run-if-empty external/coding-conventions/bin/format \"\$@\" ") add_custom_target( format-pr COMMAND bash ${CMAKE_CURRENT_BINARY_DIR}/format-pr.sh - COMMENT "Format only files modified with respect to master branch." + COMMENT "Format only files modified with respect to main/master (existing files only)" WORKING_DIRECTORY ${PROJECT_SOURCE_DIR}) # ============================================================================= @@ -1313,6 +1367,10 @@ if(NRN_ENABLE_CORENEURON) message(STATUS " PATH | ${CORENEURON_DIR}") message(STATUS " LINK FLAGS | ${CORENRN_LIB_LINK_FLAGS}") endif() +message(STATUS "NEURON GPU | ${NRN_ENABLE_GPU}") +if(NRN_ENABLE_GPU) + message(STATUS " Backend | ${NRN_GPU_BACKEND}") +endif() message(STATUS "NMODL codegen | ${NRN_ENABLE_NMODL}") if(NRN_UNIVERSAL2_BUILD) message(STATUS "CMAKE_OSX_ARCH| ${CMAKE_OSX_ARCHITECTURES}") diff --git a/bin/nrnivmodl.in b/bin/nrnivmodl.in index d66908e32b..f189ae3e36 100755 --- a/bin/nrnivmodl.in +++ b/bin/nrnivmodl.in @@ -49,6 +49,7 @@ UserLDFLAGS="" UserCOREFLAGS=() UserNMODLBIN="" UserNMODLFLAGS="" +ForceNOCMODL=false # - options come first but can be in any order. while [ "$1" ] ; do @@ -72,11 +73,14 @@ while [ "$1" ] ; do shift shift;; -nmodl) - echo "[NMODL][warning] Code generation with NMODL is pre-alpha, lacks features and is intended only for development use" + @NRN_ENABLE_GPU_FALSE@echo "[NMODL][warning] Code generation with NMODL is pre-alpha, lacks features and is intended only for development use" UserNMODLBIN="$2" UserNMODLFLAGS="--neuron $UserNMODLFLAGS" shift shift;; + -nocmodl) + ForceNOCMODL=true + shift;; -nmodlflags) echo "[NMODL][warning] If sympy is enabled, NMODL needs to be found in PYTHONPATH" UserNMODLFLAGS="$UserNMODLFLAGS $2" @@ -88,8 +92,9 @@ while [ "$1" ] ; do echo " -coreneuron Compile MOD files for CoreNEURON using nrnivmodl-core." echo " -incflags \"include flags\" Extra include flags and paths when MOD (C++) files are compiled." echo " -loadflags \"link flags\" Extra link flags, paths, and libraries when MOD (C++) files are linked." - echo " -nmodl /path/to/nmodl Path to the new NMODL transpiler for MOD files (pre-alpha, development use only)." - echo " -nmodlflags \"CLI flags\" Additional CLI flags for new NMODL transpiler" + echo " -nmodl /path/to/nmodl Path to the NMODL transpiler for MOD files." + echo " -nmodlflags \"CLI flags\" Additional CLI flags for NMODL transpiler" + echo " -nocmodl Force NOCMODL codegen (not supported with NRN_ENABLE_GPU builds)" echo " -h, --help Show this help message and exit." echo "If no MOD files or directories provided then MOD files from current directory are used." exit 0;; @@ -101,6 +106,15 @@ while [ "$1" ] ; do esac done +@NRN_ENABLE_GPU_TRUE@if [ "$ForceNOCMODL" = true ]; then +@NRN_ENABLE_GPU_TRUE@ echo "ERROR: -nocmodl is not supported with NRN_ENABLE_GPU builds. Use NMODL (default) instead." +@NRN_ENABLE_GPU_TRUE@ exit 1 +@NRN_ENABLE_GPU_TRUE@fi +@NRN_ENABLE_GPU_TRUE@if [ -n "$UserNMODLBIN" ] && echo "$UserNMODLBIN" | grep -q nocmodl; then +@NRN_ENABLE_GPU_TRUE@ echo "ERROR: NOCMODL cannot be used with NRN_ENABLE_GPU builds. Use NMODL (default) instead." +@NRN_ENABLE_GPU_TRUE@ exit 1 +@NRN_ENABLE_GPU_TRUE@fi + echo "$PWD" # Mod file paths may contain spaces which make variable lists of those diff --git a/cmake/BuildOptionDefaults.cmake b/cmake/BuildOptionDefaults.cmake index 2275e93827..3c9c980d55 100644 --- a/cmake/BuildOptionDefaults.cmake +++ b/cmake/BuildOptionDefaults.cmake @@ -16,6 +16,8 @@ set(NRN_ENABLE_MUSIC_DEFAULT OFF) set(NRN_ENABLE_RX3D_DEFAULT ON) set(NRN_ENABLE_NMODL_DEFAULT OFF) set(NRN_ENABLE_CORENEURON_DEFAULT OFF) +set(NRN_ENABLE_GPU_DEFAULT OFF) +set(NRN_GPU_BACKEND_DEFAULT "OpenACC") set(NRN_ENABLE_BACKTRACE_DEFAULT OFF) set(NRN_ENABLE_TESTS_DEFAULT OFF) set(NRN_ENABLE_MODEL_TESTS_DEFAULT "") @@ -61,6 +63,8 @@ set(NRN_OPTION_NAME_LIST NRN_ENABLE_MPI NRN_ENABLE_RX3D NRN_ENABLE_CORENEURON + NRN_ENABLE_GPU + NRN_GPU_BACKEND NRN_ENABLE_TESTS NRN_ENABLE_MODEL_TESTS NRN_ENABLE_PYTHON_DYNAMIC diff --git a/cmake/ConfigFileSetting.cmake b/cmake/ConfigFileSetting.cmake index 2c42b0a722..882244b0c1 100644 --- a/cmake/ConfigFileSetting.cmake +++ b/cmake/ConfigFileSetting.cmake @@ -42,6 +42,14 @@ else() set(CORENEURON_ENABLED_FALSE "") endif() +if(NRN_ENABLE_GPU) + set(NRN_ENABLE_GPU_TRUE "") + set(NRN_ENABLE_GPU_FALSE "#") +else() + set(NRN_ENABLE_GPU_TRUE "#") + set(NRN_ENABLE_GPU_FALSE "") +endif() + # ~~~ # A variable that doesn't start out as #undef but as #define needs an # explicit @...@ replacement in the .h.in files. diff --git a/cmake/NeuronTestHelper.cmake b/cmake/NeuronTestHelper.cmake index 311cf9fec8..e6857b4352 100644 --- a/cmake/NeuronTestHelper.cmake +++ b/cmake/NeuronTestHelper.cmake @@ -209,6 +209,9 @@ function(nrn_add_test_group) endforeach() # Construct the names of the important output files set(special "${nrnivmodl_directory}/${CMAKE_HOST_SYSTEM_PROCESSOR}/special") + set(nrnmech_lib + "${nrnivmodl_directory}/${CMAKE_HOST_SYSTEM_PROCESSOR}/${CMAKE_SHARED_LIBRARY_PREFIX}nrnmech${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) # Add the custom command to generate the binaries. Get nrnivmodl from the build directory. At # the moment it seems that `nrnivmodl` is generated at configure time, so there is no target # to depend on and it should always be available, but it will try and link against libnrniv.so @@ -217,6 +220,11 @@ function(nrn_add_test_group) # be a wrapper that invokes CMake? set(output_binaries "${special}") list(APPEND nrnivmodl_dependencies nrniv_lib) + # Re-run nrnivmodl when libnrniv.so changes so test special does not retain stale NVHPC + # pgcudafat object paths from a prior libnrniv link line. + list( + APPEND nrnivmodl_dependencies + "${CMAKE_BINARY_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}nrniv${CMAKE_SHARED_LIBRARY_SUFFIX}") if(NRN_ENABLE_CORENEURON AND NRN_ADD_TEST_GROUP_CORENEURON) list(APPEND output_binaries "${special}-core") if((NOT coreneuron_FOUND) AND (NOT DEFINED CORENEURON_BUILTIN_MODFILES)) @@ -229,6 +237,8 @@ function(nrn_add_test_group) add_custom_command( OUTPUT ${output_binaries} DEPENDS ${nrnivmodl_dependencies} ${modfile_build_paths} + # Force libnrnmech relink when libnrniv.so changes (NVHPC embeds transient pgcudafat paths). + COMMAND ${CMAKE_COMMAND} -E rm -f ${nrnmech_lib} ${special} COMMAND ${nrnivmodl_command} COMMENT "Building special[-core] for test group ${NRN_ADD_TEST_GROUP_NAME}" WORKING_DIRECTORY "${nrnivmodl_directory}") @@ -281,7 +291,11 @@ function(nrn_add_test) else() set(feature_mod_compatibility_enabled OFF) endif() - set(feature_gpu_enabled ${CORENRN_ENABLE_GPU}) + if(NRN_ENABLE_GPU OR (NRN_ENABLE_CORENEURON AND CORENRN_ENABLE_GPU)) + set(feature_gpu_enabled ON) + else() + set(feature_gpu_enabled OFF) + endif() # Check REQUIRES set(requires_coreneuron OFF) foreach(required_feature ${NRN_ADD_TEST_REQUIRES}) @@ -339,6 +353,13 @@ function(nrn_add_test) if(DEFINED NRN_ADD_TEST_SIM_DIRECTORY) set(sim_directory "${NRN_ADD_TEST_SIM_DIRECTORY}") endif() + # NVHPC may leave transient fat-object files in $TMPDIR when special runs with -gpu under MPI. Use + # a per-test TMPDIR under the build tree (see external_ringtest GPU ctest notes). + if("gpu" IN_LIST NRN_ADD_TEST_REQUIRES AND (CORENRN_ENABLE_GPU OR NRN_ENABLE_GPU)) + set(gpu_tmpdir "${PROJECT_BINARY_DIR}/test/tmp/${NRN_ADD_TEST_GROUP}/${NRN_ADD_TEST_NAME}") + file(MAKE_DIRECTORY "${gpu_tmpdir}") + list(APPEND extra_environment "TMPDIR=${gpu_tmpdir}") + endif() # Finally a working directory for this specific test within the group set(working_directory "${PROJECT_BINARY_DIR}/test/${NRN_ADD_TEST_GROUP}/${NRN_ADD_TEST_NAME}") file(MAKE_DIRECTORY "${working_directory}") @@ -405,9 +426,15 @@ function(nrn_add_test) if(NOT "PATH" IN_LIST test_env_var_names) message(FATAL_ERROR "Expected to find PATH in ${test_env_var_names} but didn't") endif() + set(_nrn_test_mech_dir "${nrnivmodl_directory}/${CMAKE_HOST_SYSTEM_PROCESSOR}") # PATH will already be set in test_env - list(TRANSFORM test_env REPLACE "^PATH=" - "PATH=${nrnivmodl_directory}/${CMAKE_HOST_SYSTEM_PROCESSOR}:") + list(TRANSFORM test_env REPLACE "^PATH=" "PATH=${_nrn_test_mech_dir}:") + # pytest/python3 launches do not search PATH for dlopen(libnrnmech.so). + if("LD_LIBRARY_PATH" IN_LIST test_env_var_names) + list(TRANSFORM test_env REPLACE "^LD_LIBRARY_PATH=" "LD_LIBRARY_PATH=${_nrn_test_mech_dir}:") + else() + list(APPEND test_env "LD_LIBRARY_PATH=${_nrn_test_mech_dir}") + endif() endif() list(TRANSFORM test_env REPLACE "^PYTHONPATH=" "PYTHONPATH=${CMAKE_SOURCE_DIR}/docs/nmodl/python_scripts:") diff --git a/cmake/coreneuron/OpenAccHelper.cmake b/cmake/coreneuron/OpenAccHelper.cmake index 6df39ad936..0d7acc8ce3 100644 --- a/cmake/coreneuron/OpenAccHelper.cmake +++ b/cmake/coreneuron/OpenAccHelper.cmake @@ -96,6 +96,7 @@ if(CORENRN_ENABLE_GPU) # something like `-acc -lcorenrnmech ...`. CORENRN_NEURON_LINK_FLAGS only contains flags that need # to be used when linking the NEURON Python module to make sure it is able to dynamically load # libcorenrnmech.so. + set_property(GLOBAL PROPERTY CORENRN_ACC_COMP_FLAGS "${NVHPC_ACC_COMP_FLAGS}") set_property(GLOBAL PROPERTY CORENRN_LIB_LINK_FLAGS "${NVHPC_ACC_COMP_FLAGS}") if(CORENRN_ENABLE_SHARED) # Because of diff --git a/cmake/neuronMechMaker.cmake b/cmake/neuronMechMaker.cmake index ece62bbf43..97c2f69dfa 100644 --- a/cmake/neuronMechMaker.cmake +++ b/cmake/neuronMechMaker.cmake @@ -98,7 +98,7 @@ API reference list of mod files to convert. ``NMODL_NEURON_EXTRA_ARGS`` - (*optional*, default: None) list of additional arguments to pass to NMODL for NEURON codegen. + (*optional*, default: ``passes --inline host --c`` if CUDA disabled, ``passes --inline host --c acc --oacc`` if ``NRN_ENABLE_GPU=ON`` and CUDA enabled) list of additional arguments to pass to NMODL for NEURON codegen. ``NMODL_CORENEURON_EXTRA_ARGS`` (*optional*, default: ``passes --inline host --c`` if CUDA disabled, ``passes --inline host --c acc --oacc`` if CUDA enabled) list of additional arguments to pass to NMODL for coreNEURON codegen. @@ -260,6 +260,16 @@ function(create_nrnmech) message("${MESSAGE_PRIORITY}" "LIBRARY_TYPE | ${LIBRARY_TYPE}") + # GPU builds require NMODL NEURON codegen (NOCMODL cannot emit OpenACC for libnrnmech). + if(NRN_MECH_NEURON AND NRN_ENABLE_GPU) + if(NOT NRN_MECH_NMODL_NEURON_CODEGEN) + set(NRN_MECH_NMODL_NEURON_CODEGEN ON) + message( + "${MESSAGE_PRIORITY}" + "NRN_ENABLE_GPU=ON: defaulting NEURON mechanism codegen to NMODL (NMODL_NEURON_CODEGEN)") + endif() + endif() + # nmodl by default generates code for coreNEURON, so we toggle this via an option if(NRN_MECH_NMODL_NEURON_CODEGEN) set(NEURON_TRANSPILER_LAUNCHER ${NMODL_EXECUTABLE} --neuron) @@ -271,6 +281,12 @@ function(create_nrnmech) if(NRN_MECH_NEURON AND NOT NRN_MECH_NMODL_NEURON_CODEGEN AND NRN_MECH_NMODL_NEURON_EXTRA_ARGS) + if(NRN_ENABLE_GPU) + message( + FATAL_ERROR + "${CMAKE_CURRENT_FUNCTION}: NOCMODL NEURON codegen is not supported when NRN_ENABLE_GPU=ON." + ) + endif() message( WARNING "${CMAKE_CURRENT_FUNCTION}: requested NEURON library with NOCMODL codegen, but NMODL_NEURON_EXTRA_ARGS is not empty; " @@ -280,6 +296,14 @@ function(create_nrnmech) set(NRN_MECH_NMODL_NEURON_EXTRA_ARGS) endif() + # set default flags for NMODL for NEURON mechanism codegen. + if(NOT NRN_MECH_NMODL_NEURON_EXTRA_ARGS) + set(NRN_MECH_NMODL_NEURON_EXTRA_ARGS passes --inline host --c) + if(NRN_ENABLE_GPU AND CMAKE_CUDA_COMPILER) + list(APPEND NRN_MECH_NMODL_NEURON_EXTRA_ARGS acc --oacc) + endif() + endif() + list(JOIN NRN_MECH_NMODL_NEURON_EXTRA_ARGS "" NMODL_NEURON_EXTRA_ARGS_SPACES) message("${MESSAGE_PRIORITY}" "NMODL_NEURON_EXTRA_ARGS | ${NMODL_NEURON_EXTRA_ARGS_SPACES}") @@ -441,6 +465,41 @@ function(create_nrnmech) "${EXECUTABLE_OUTPUT_DIR}") endif() + # Mirror CoreNEURON GPU link treatment on NEURON libnrnmech/special so dynamically loaded + # OpenACC mechanisms work when special is launched with -coreneuron -gpu (see NVIDIA forum + # thread on loading OpenACC shared libraries from NVHPC-linked executables). + if(CMAKE_CUDA_COMPILER AND (CORENRN_ENABLE_GPU OR NRN_ENABLE_GPU)) + if(NOT CUDAToolkit_FOUND) + find_package(CUDAToolkit 9.0 REQUIRED) + endif() + if(NOT OpenACC_FOUND) + find_package(OpenACC REQUIRED) + endif() + if("${LIBRARY_TYPE}" STREQUAL "STATIC") + target_link_libraries(${TARGET_LIBRARY_NAME} PUBLIC CUDA::cudart_static + OpenACC::OpenACC_CXX) + elseif("${LIBRARY_TYPE}" STREQUAL "SHARED") + target_link_libraries(${TARGET_LIBRARY_NAME} PUBLIC CUDA::cudart OpenACC::OpenACC_CXX) + else() + message(FATAL_ERROR "Unsupported library type for CUDA: ${LIBRARY_TYPE}") + endif() + # Full OpenACC link line (not only -cuda) so special can load libnrniv OpenACC fat objects + # after GPU adoption step 3 propagates -acc compile to cellorder in libnrniv. + set(_nrn_mech_gpu_link_flags "-cuda") + if(CORENRN_NEURON_LINK_FLAGS) + set(_nrn_mech_gpu_link_flags ${CORENRN_NEURON_LINK_FLAGS}) + else() + get_property(_nrn_acc_comp_flags GLOBAL PROPERTY CORENRN_ACC_COMP_FLAGS) + if(_nrn_acc_comp_flags) + separate_arguments(_nrn_mech_gpu_link_flags UNIX_COMMAND "${_nrn_acc_comp_flags}") + endif() + endif() + target_link_options(${TARGET_LIBRARY_NAME} PUBLIC ${_nrn_mech_gpu_link_flags}) + if(NRN_MECH_SPECIAL) + target_link_options(${TARGET_EXECUTABLE_NAME} PUBLIC ${_nrn_mech_gpu_link_flags}) + endif() + endif() + endif() # Convert mod files for use with coreNEURON diff --git a/cmake/nrnivmodl.cmake b/cmake/nrnivmodl.cmake index bb9fb8a405..76d61f7495 100644 --- a/cmake/nrnivmodl.cmake +++ b/cmake/nrnivmodl.cmake @@ -30,6 +30,9 @@ endif() if(NRNIVMODL_SPECIAL) list(APPEND NRNIVMODL_ARGS "SPECIAL") endif() +if(@NRN_ENABLE_GPU@) + list(APPEND NRNIVMODL_ARGS "NMODL_NEURON_CODEGEN") +endif() message(STATUS "Received mod files: ${NRNIVMODL_MOD_FILES}") diff --git a/docs/cmake_doc/options.rst b/docs/cmake_doc/options.rst index ca1d078769..ce1bf0803f 100644 --- a/docs/cmake_doc/options.rst +++ b/docs/cmake_doc/options.rst @@ -391,6 +391,53 @@ NRN_RX3D_OPT_LEVEL:STRING=0 -DNRN_RX3D_OPT_LEVEL=2 +NEURON GPU options +================== + +.. _cmake-nrn-enable-gpu-option: + +NRN_ENABLE_GPU:BOOL=OFF +----------------------- + Enable NEURON native GPU support (Phase B: fixed-step ``pc.psolve``). + + This is the user-facing CMake option for GPU acceleration in NEURON proper. + With ``NRN_ENABLE_GPU=ON``, models can run on ``gpu.backend="native"`` without + embedding CoreNEURON for fixed-step integration. Scope, limitations, and + runtime API are documented in :doc:`/dev/native-gpu-fixed-step`. + + GPU development builds still typically enable CoreNEURON and + ``CORENRN_ENABLE_GPU`` for the broader CTest GPU suite and mechanism parity + tests. + + A typical GPU development build (NVHPC required) uses: + + .. code-block:: shell + + cmake .. -G Ninja \ + -DCMAKE_INSTALL_PREFIX=install \ + -DCMAKE_BUILD_TYPE=RelWithDebInfo \ + -DNRN_ENABLE_CORENEURON=ON \ + -DNRN_ENABLE_GPU=ON \ + -DCORENRN_ENABLE_GPU=ON \ + -DCMAKE_C_COMPILER=nvc \ + -DCMAKE_CXX_COMPILER=nvc++ \ + -DCMAKE_CUDA_COMPILER=nvcc \ + -DCMAKE_CUDA_ARCHITECTURES=75 + + ``CMAKE_CUDA_ARCHITECTURES`` is **optional** but recommended: set it to your + GPU's compute capability (e.g. ``75`` for NVIDIA Turing / T1000). If omitted, + CoreNEURON defaults to ``70`` and ``80`` (Volta + Ampere), which usually + still runs on 7.x GPUs but compiles more code than a single-arch dev build + needs. See ``~/neuron/notes/gpu_workstation.md`` for a workstation-specific + recipe. + +NRN_GPU_BACKEND:STRING=OpenACC +------------------------------ + Select the GPU backend for NEURON native GPU. + + Only ``OpenACC`` is supported in Phase A-B. Other values cause configure to + fail when ``NRN_ENABLE_GPU=ON``. + CoreNEURON options ================== @@ -399,7 +446,9 @@ NRN_ENABLE_CORENEURON:BOOL=OFF Enable CoreNEURON support If ON CoreNEURON will be built and any needed NMODL submodule dependencies - cloned as external submodules. + cloned as external submodules. For new GPU work, prefer enabling + ``NRN_ENABLE_GPU`` in addition to (or eventually instead of) relying on + CoreNEURON as the only GPU entry point. NRN_ENABLE_MOD_COMPATIBILITY:BOOL=OFF ------------------------------------- @@ -412,10 +461,15 @@ NRN_ENABLE_MOD_COMPATIBILITY:BOOL=OFF Other CoreNEURON options: ------------------------- - There are 20 or so cmake arguments specific to a CoreNEURON + +.. _cmake-coreneuron-enable-gpu-option: + + ``CORENRN_ENABLE_GPU:BOOL=OFF`` — enable OpenACC GPU execution in CoreNEURON. + Required for CoreNEURON GPU tests; native GPU modtests use ``gpu.backend=native`` + (see :doc:`/dev/native-gpu-fixed-step`). + + There are 20 or so other cmake arguments specific to a CoreNEURON build that are listed in https://github.com/neuronsimulator/nrn/blob/master/src/coreneuron/CMakeLists.txt. - The one of particular interest that can be used on the NEURON - CMake configure line is `CORENRN_ENABLE_GPU`. NMODL options ============= diff --git a/docs/dev/gpu-testing.rst b/docs/dev/gpu-testing.rst index 87ecdf2011..ba309613f7 100644 --- a/docs/dev/gpu-testing.rst +++ b/docs/dev/gpu-testing.rst @@ -1,5 +1,9 @@ Testing GPU functionality ######################### + +For the **Phase B scope contract** (supported features, timestep architecture, +``neuron.gpu`` API), see :doc:`native-gpu-fixed-step`. + This section provides information and links that help with testing :ref:`CoreNEURON`'s GPU support. Other sections of the documentation that may be relevant are: @@ -62,15 +66,81 @@ You can filter which tests are run by name using the ``-R`` option to CTest, for .. code-block:: console + $ ctest -N -R gpu # list GPU tests (count varies with build options) $ ctest --output-on-failure -R gpu Test project /path/to/your/build Start 42: coreneuron_modtests::direct_py_gpu - 1/53 Test #42: coreneuron_modtests::direct_py_gpu ............................. Passed 1.98 sec + 1/68 Test #42: coreneuron_modtests::direct_py_gpu ............................. Passed 1.98 sec Start 43: coreneuron_modtests::direct_hoc_gpu - 2/53 Test #43: coreneuron_modtests::direct_hoc_gpu ............................ Passed 1.03 sec + 2/68 Test #43: coreneuron_modtests::direct_hoc_gpu ............................ Passed 1.03 sec Start 44: coreneuron_modtests::spikes_py_gpu ... +A full MPI + GPU + tests build typically registers **about 96** tests matching ``-R gpu`` +(including 40 ``coreneuron_modtests::*_py_gpu*`` variants and **19** ``*_py_gpu_native`` +native-backend modtests). Use ``ctest -N -R gpu`` in your build tree for the +authoritative count. + +GPU-related CMake options: + +- ``-DNRN_ENABLE_GPU=ON`` — user-facing NEURON native GPU option (Phase B fixed-step path) +- ``-DCORENRN_ENABLE_GPU=ON`` — enables OpenACC GPU execution in CoreNEURON (required for CoreNEURON GPU tests) +- ``-DNRN_GPU_BACKEND=OpenACC`` — only supported backend in Phase A-B + +See :ref:`cmake-nrn-enable-gpu-option` and :ref:`cmake-coreneuron-enable-gpu-option` in the CMake options documentation. + +NEURON native GPU backend (Phase B) +*********************************** +See :doc:`native-gpu-fixed-step` for runtime API, supported/unsupported matrix, +and timestep architecture. This section covers CTest and MPI device assignment. + +MPI multi-GPU device assignment uses ``device_id = mpi_local_rank % num_gpus_per_node`` +(same policy as CoreNEURON ``init_gpu()``). Set ``gpu.device_count`` to limit GPUs per node +(0 = all available). CTest: ``unit_tests::gpu_device_assign_mpi`` (2 MPI ranks). + +The **G4 native modtest parity** ctests set ``NRN_GPU_BACKEND_TEST=native`` and compare +NEURON CPU reference output against the native backend (19 tests, mirroring single-process +``*_py_gpu`` CoreNEURON modtests): + +.. code-block:: console + + $ ctest -N -R '_py_gpu_native' + $ ctest --output-on-failure -R '_py_gpu_native' + +Ringtest native GPU benchmark +***************************** +The external ringtest script supports ``-gpu-native`` for NEURON native GPU runs +(orthogonal to CoreNEURON ``-gpu``): + +.. code-block:: console + + $ cd build/test/external_ringtest/neuron_gpu_native_mpi + $ special -mpi -python ringtest.py -gpu-native -tstop 100 + +The script prints ``runtime=``, ``load_balance=``, and ``spk_time`` statistics. +Compare spike output with ``sortspike`` against ``reference_data/spk1.100ms.std.ref``: + +.. code-block:: console + + $ sortspike spk1.std > spk1.srt + $ diff spk1.srt ../../external/tests/ringtest/reference_data/spk1.100ms.std.ref + +CTest: ``external_ringtest::neuron_gpu_native_mpi`` (MPI, 100 ms, spike comparison group). + +The ``external/tests/ringtest`` tree is gitignored in the NEURON repo; apply +``test/external/ringtest/ringtest-gpu-native.patch`` inside your local ringtest +clone (under ``external/tests/ringtest``) before running manual ringtest or +reconfiguring CMake. + +Ringtest GPU ctest note +*********************** +The ``external_ringtest::coreneuron_gpu_mpi`` test historically failed on some NVHPC builds with +``pgcudafat*.o: cannot open shared object file`` when launching NEURON ``special`` with ``-gpu`` +under MPI. The ctest harness now runs the GPU MPI ringtest via ``special-core`` (dump model with +``special``, simulate with ``special-core --gpu``), matching the offline ringtest pattern. +GPU tests also use a per-test ``TMPDIR`` under the build tree to avoid transient NVHPC loader +issues. + Running tests manually ********************** It is sometimes convenient to run basic tests outside the CTest diff --git a/docs/dev/index.rst b/docs/dev/index.rst index 97d1a798cb..6ff5c3305c 100644 --- a/docs/dev/index.rst +++ b/docs/dev/index.rst @@ -7,6 +7,8 @@ NEURON Development topics ./HOCInterpreter/HOCInterpreter.md ./how-do-i/how-do-i.rst data-structures.rst + native-gpu-fixed-step.rst + ./native-gpu-adoption/00-overview.md gpu-testing.rst nmodl-development.rst workflow-code-paths.rst diff --git a/docs/dev/native-gpu-adoption/00-overview.md b/docs/dev/native-gpu-adoption/00-overview.md new file mode 100644 index 0000000000..2e765f33b6 --- /dev/null +++ b/docs/dev/native-gpu-adoption/00-overview.md @@ -0,0 +1,68 @@ +# Native GPU adoption — overview (Phase B) + +Development journal for NEURON's native fixed-step GPU path (`gpu.backend="native"`). +The published scope contract lives in Sphinx: +[`native-gpu-fixed-step.rst`](../native-gpu-fixed-step.rst). + +## Goals (Phase B) + +- Fixed-step `pc.psolve` / `h.fadvance` on the native GPU path without embedding + CoreNEURON at **runtime** (`coreneuron.enable=False`). +- OpenACC matrix solve, mechanism integration, and post-solve on device. +- CPU spike priority queues + MPI at the minimum NetCon delay interval; GPU + `NET_RECEIVE` computation and `net_send` buffering. +- Parity proof: **19/19** `*_py_gpu_native` modtests (mirror single-process + `*_py_gpu` CoreNEURON modtests). +- Explicit boundaries: CVode rejected, documented threading and spike policy. + +## Non-goals (deferred past Phase B) + +See [04-known-limitations.md](04-known-limitations.md) and +[05-future-work.md](05-future-work.md). Phase B does **not** include: + +- CVode / IDA on native GPU +- GPU spike priority queue +- Standalone `-DNRN_ENABLE_GPU=ON` configure without `-DNRN_ENABLE_CORENEURON=ON` +- Full multi-thread OpenACC maturity +- Device partrans gather/scatter (gap models use CPU fixed-step fallback today) +- MPI-native modtest expansion beyond the current parity set + +## Stopping point + +Phase B is **complete** when the runtime contract is coherent for fixed-step +`pc.psolve` (L0–L8; see [02-implementation-map.md](02-implementation-map.md)) +and the checklist in [PHASE-B-COMPLETE.md](PHASE-B-COMPLETE.md) is satisfied. + +Branch: `hines-grok/feature/neuron-core-gpu-adoption` +Commits through PR7: `2087fb131` (appendix: [appendix-commits.md](appendix-commits.md)). + +## Document map + +| File | Purpose | +|------|---------| +| [01-design-decisions.md](01-design-decisions.md) | Why spike queues stay on CPU, Memb_list padding, etc. | +| [02-implementation-map.md](02-implementation-map.md) | L0–L8 layers → files | +| [03-test-parity.md](03-test-parity.md) | `_py_gpu` vs `_py_gpu_native` | +| [04-known-limitations.md](04-known-limitations.md) | Honest runtime limits | +| [05-future-work.md](05-future-work.md) | Deferred items (no Phase B scope creep) | +| [appendix-commits.md](appendix-commits.md) | One paragraph per adoption commit | +| [PHASE-B-COMPLETE.md](PHASE-B-COMPLETE.md) | Closure checklist | + +## Conceptual levels (L0–L8) + +Use these as lenses when reading code, tests, or PRs: + +| Level | Name | Summary | +|-------|------|---------| +| L0 | Build | CMake, NVHPC, OpenACC; `NRN_ENABLE_GPU` | +| L1 | Runtime API | `neuron.gpu`, `ParallelContext` GPU methods | +| L2 | Device lifecycle | Upload once; invalidate on model change | +| L3 | Timestep dispatch | CPU vs native routing in `fadvance` | +| L4 | Integration step | Matrix, mechanisms, post-solve | +| L5 | Host fallbacks | sparse13, extracellular, LFP | +| L6 | Communication | Spike queues (CPU), gap MPI, `NetSendBuffer` | +| L7 | Recording | Batch download / flush interval | +| L8 | Verification | Modtests, unit tests, ringtest | + +Phase B “done” means L0–L8 are coherent for fixed-step native `pc.psolve` with +the documented exclusions above. \ No newline at end of file diff --git a/docs/dev/native-gpu-adoption/01-design-decisions.md b/docs/dev/native-gpu-adoption/01-design-decisions.md new file mode 100644 index 0000000000..8ec6b50ed8 --- /dev/null +++ b/docs/dev/native-gpu-adoption/01-design-decisions.md @@ -0,0 +1,121 @@ +# Design decisions + +Rationale for choices that are not obvious from the Sphinx scope contract alone. + +## Spike delivery stays on CPU; `NET_RECEIVE` runs on GPU + +**Decision:** Cross-rank NetCon scheduling uses CPU priority queues and MPI +exchange at the minimum delay interval. `NET_RECEIVE` mechanism code and +integration run on GPU during the fixed step. + +**Why:** Matches the CoreNEURON split already proven at scale. A GPU-resident +spike queue would require new ordering guarantees across MPI ranks and +device/host event ordering. Phase B reuses the host network stack. + +**Artifacts:** `net_events.cpp`, `net_events.hpp`, `net_send_buffer.cpp`, +`deliver_net_events` / `deliver_post_step_events_host` in `fadvance_gpu.cpp`. + +## `net_send` / threshold: GPU buffer, CPU/MPI for cross-cell + +**Decision:** Presynaptic `net_send` and threshold crossings generated on device +are buffered (`NetSendBuffer`). Self-events may flush on device; cross-cell +outputs go to CPU/MPI scheduling. + +**Why:** Preserves existing NetCon semantics without a device-side priority +queue. Buffer flush points are tied to the fixed-step cadence. + +## Memb_list padding for device upload + +**Decision:** Upload padded `Memb_list` shells so OpenACC mechanism kernels see +a uniform SOA layout on device. + +**Why:** Host `Memb_list` layout includes holes and permutations that do not map +1:1 to GPU warp-friendly access. Padding mirrors CoreNEURON's device layout +assumptions. + +**Artifacts:** `upload_mechanisms.cpp`, `upload.cpp`. + +## `ensure_cached` / device_token lifecycle + +**Decision:** Device state uploads once per sorted model token; concurrent +upload paths guard with caching (regression fix for double-free in gap/natrans +tests). + +**Why:** Re-uploading on every step would dominate cost; stale device state +after topology changes must invalidate cleanly. + +**Artifacts:** `device_state.cpp`, `upload.cpp`. + +## Gap junctions: CPU fixed-step fallback in Phase B + +**Decision:** When `pc.setup_transfer()` registers `nrnthread_v_transfer_`, +native GPU fixed-step dispatches the **full CPU body** in `nrn_fixed_step_thread` +(`fadvance.cpp`) instead of `fadvance_gpu.cpp`. Partrans then runs unchanged on +the CPU path: + +1. Gather sources into buffers (`mpi_transfer` / `outsrc_buf_`) +2. `MPI_Alltoallv` when `nrnmpi_numprocs > 1` (skipped on one rank) +3. Scatter to targets (`thread_transfer` in `nonvint`) + +`gpu.enable=True` still uploads model state; integration per step is CPU until +device-resident gather/scatter is implemented. + +**Why:** A hybrid GPU solve plus host partrans gather/scatter diverged on Traub +(`use_gap=1`) and could overwrite host voltages with stale device state. Full CPU +fallback restores raster parity (`test_par_gj_native_gpu.py`, ringtest `-gap`) at +~1× NEURON CPU cost. Device partrans buffers are follow-up work, not Phase B. + +**Artifacts:** `fadvance.cpp`, `partrans.cpp`, `sync.cpp`, `fadvance_gpu.cpp`. + +## Batch download for recording + +**Decision:** `download_flush_interval` controls how often post-solve state is +pulled to host for `Vector.record` and HOC graph reads. Default `1` (every +step); `0` defers to end of `psolve`. + +**Why:** Per-step host pulls are correct but expensive. Batch API lets models +trade accuracy of mid-solve visibility for throughput. + +**Artifacts:** `download.cpp`, `post_solve.cpp`, `neuron.gpu.download_flush_interval`. + +## CVode rejected at enable time + +**Decision:** `native_gpu_configuration_error()` blocks native GPU when CVode is +active (Python `gpu.enable`, HOC `gpu_enable`, entry to `nrn_fixed_step`). + +**Why:** Variable-step integration is out of Phase B scope; silent fallback to +CPU would hide misconfiguration. + +**Artifacts:** `config.cpp`, `gpu.py`, `ocbbs.cpp`, `fadvance.cpp`. + +## Threading: warn, do not block + +**Decision:** `pc.nthread(n>1)` with native GPU emits a one-time warning; modtests +run with `OMP_NUM_THREADS=1` and `pc.nthread(1)`. + +**Why:** Per-thread OpenACC CUDA contexts are fragile on some platforms. Blocking +would break existing scripts; warning sets expectations until a later hardening +pass. + +**Artifacts:** `warn_native_gpu_multithread_policy()` in `config.cpp`, +`fadvance_gpu.cpp`. + +## Subphase GPU path; tree solve vs sparse13 + +**Decision:** Native fixed-step is split into GPU subphases (matrix setup, tree +solve, post-solve, `NET_RECEIVE`, `net_send` buffer, …). ODE tree models use +**GPU tree solve** (`solve_interleaved`). DAE models (extracellular, +`LinearMechanism`) swap only the **matrix solve** to CPU `sparse13` +(`spFactor`/`spSolve`); other subphases are not architecturally excluded. + +**Why:** Sparse Gaussian elimination is the missing GPU piece — a future phase, +like CVode-on-GPU. Phase B validates the ODE column only. + +## Build still requires CoreNEURON at configure (Phase B) + +**Decision:** `-DNRN_ENABLE_GPU=ON` currently requires `-DNRN_ENABLE_CORENEURON=ON` +at CMake configure. Runtime native path does not require `coreneuron.enable=True`. + +**Why:** OpenACC flags, cellorder sources, and mechanism GPU codegen still flow +through the CoreNEURON configure path. Standalone GPU configure is explicitly +**out of Phase B** (see [05-future-work.md](05-future-work.md)). \ No newline at end of file diff --git a/docs/dev/native-gpu-adoption/02-implementation-map.md b/docs/dev/native-gpu-adoption/02-implementation-map.md new file mode 100644 index 0000000000..5e0d6a9997 --- /dev/null +++ b/docs/dev/native-gpu-adoption/02-implementation-map.md @@ -0,0 +1,92 @@ +# Implementation map (L0–L8) + +File → responsibility guide for the native GPU path. Paths are relative to the +repo root. + +## L0 — Build + +| Artifact | Role | +|----------|------| +| `CMakeLists.txt` | `NRN_ENABLE_GPU`, CoreNEURON coupling guard | +| `src/neuron/CMakeLists.txt` | `neuron_gpu`, `neuron_gpu_upload` targets | +| `src/nrniv/CMakeLists.txt` | OpenACC cellorder sources linked into `libnrniv` | +| `cmake/coreneuron/OpenAccHelper.cmake` | NVHPC `-acc` compile/link flags | +| `cmake/neuronMechMaker.cmake` | OpenACC mechanism object builds | +| `docs/cmake_doc/options.rst` | User-facing CMake docs | + +## L1 — Runtime API + +| Artifact | Role | +|----------|------| +| `share/lib/python/neuron/gpu.py` | `neuron.gpu` module; CVode guard on enable | +| `share/lib/python/neuron/coreneuron.py` | Legacy `coreneuron.gpu` → `neuron.gpu` | +| `src/parallel/ocbbs.cpp` | HOC `gpu_enable`, `gpu_backend`, `gpu_device_count`, `gpu_download_flush_interval` | +| `src/neuron/gpu/config.cpp` | C++ runtime config; `native_gpu_configuration_error()` | + +## L2 — Device lifecycle + +| Artifact | Role | +|----------|------| +| `src/neuron/gpu/device_state.cpp` | Device resident state, invalidate | +| `src/neuron/gpu/upload.cpp` | SOA / tree / thread upload | +| `src/neuron/gpu/upload_mechanisms.cpp` | Padded `Memb_list` upload | +| `src/neuron/gpu/offload.cpp` | OpenACC/OpenMP offload helpers | + +## L3 — Timestep dispatch + +| Artifact | Role | +|----------|------| +| `src/nrnoc/fadvance.cpp` | `nrn_fixed_step` entry; native branch in `nrn_fixed_step_thread` | +| `src/neuron/gpu/fadvance_gpu.cpp` | `fixed_step_thread`, host event delivery hooks | +| `src/parallel/ocbbs.cpp` | `psolve` → `netpar_solve` vs `nrncore_psolve` | + +## L4 — Integration step + +| Artifact | Role | +|----------|------| +| `src/nrnoc/treeset.cpp` | OpenACC `setup_tree_matrix` path | +| `src/coreneuron/permute/cellorder*.cpp` | Interleaved solve (compiled into `libnrniv`) | +| `src/neuron/gpu/post_solve.cpp` | Device post-solve: voltage, fast_imem, capacity | +| `src/neuron/gpu/fadvance_gpu.cpp` | Orchestrates deliver → matrix → solve → post-solve | + +## L5 — Host fallbacks + +| Artifact | Role | +|----------|------| +| `src/neuron/gpu/post_solve.cpp` | `post_solve_needs_host_fallback()` | +| `src/nrnoc/fadvance.cpp` | CPU `nrn_update_voltage` when fallback required; **gap CPU fixed-step fallback** when `nrnthread_v_transfer_` is set | + +## L6 — Communication + +| Artifact | Role | +|----------|------| +| `src/neuron/gpu/net_events.cpp` | Host-side net event stages | +| `src/neuron/gpu/net_events.hpp` | Spike / `NET_RECEIVE` policy comments | +| `src/neuron/gpu/net_send_buffer.cpp` | GPU `net_send` buffering | +| `src/neuron/gpu/net_send_buffer.hpp` | Buffer semantics | +| `src/neuron/gpu/sync.cpp` | Gap voltage device↔host sync (hybrid path; unused on Phase B CPU fallback) | +| `src/nrniv/partrans.cpp` | Partrans gather / MPI / scatter (CPU) | + +## L7 — Recording + +| Artifact | Role | +|----------|------| +| `src/neuron/gpu/download.cpp` | `batch_download_to_host`, flush interval | +| `src/neuron/gpu/download.hpp` | Download API | + +## L8 — Verification + +| Artifact | Role | +|----------|------| +| `test/CMakeLists.txt` | `*_py_gpu_native` modtests, GPU unit tests | +| `test/coreneuron/backend_helper.py` | Native vs CoreNEURON backend selection in tests | +| `test/unit_tests/gpu/*.cpp` | Catch2 GPU module tests | +| `test/external/ringtest/` | `-gpu-native` ringtest harness | +| `docs/dev/gpu-testing.rst` | CTest / ringtest workflows | +| `docs/dev/native-gpu-fixed-step.rst` | Phase B scope contract (PR6) | + +## Dispatch flow (one line) + +`pc.psolve` → (`coreneuron.enable` ? `nrncore_psolve` : `netpar_solve` → +`nrn_fixed_step` → (`gpu.enable && backend==native` ? `fadvance_gpu` : CPU thread +path)). \ No newline at end of file diff --git a/docs/dev/native-gpu-adoption/03-test-parity.md b/docs/dev/native-gpu-adoption/03-test-parity.md new file mode 100644 index 0000000000..810467acd1 --- /dev/null +++ b/docs/dev/native-gpu-adoption/03-test-parity.md @@ -0,0 +1,91 @@ +# Test parity + +## Native modtest suite + +Phase B verification anchor: **19** CTest targets matching `*_py_gpu_native`. + +They mirror single-process CoreNEURON `*_py_gpu` modtests with: + +```text +NRN_GPU_BACKEND_TEST=native +NRN_GPU_PERMUTE=2 +OMP_NUM_THREADS=1 +``` + +Defined in `test/CMakeLists.txt` (G4 native GPU modtest parity block). + +Quick check: + +```console +ctest -N -R '_py_gpu_native' +ctest --output-on-failure -R '_py_gpu_native' +``` + +## `_py_gpu` vs `_py_gpu_native` + +| Aspect | `*_py_gpu` | `*_py_gpu_native` | +|--------|------------|-------------------| +| Backend | CoreNEURON embedded GPU | `gpu.backend=native` | +| `coreneuron.enable` | True (typical) | False | +| Mechanism codegen | CoreNEURON OpenACC | NEURON native OpenACC (NMODL visitor) | +| Spike / MPI model | CoreNEURON | Native (CPU queues + GPU integration) | +| Reference | NEURON CPU | Same NEURON CPU reference | +| Count | ~40 single-process variants | 19 (Phase B parity set) | + +Parity means: same mod file, same CPU reference output, native backend +matches within test tolerances. + +## Broader GPU regression + +Full GPU CI (CoreNEURON + native + ringtest) uses: + +```console +ctest --output-on-failure -R gpu +``` + +Expect **~96** tests when MPI + CoreNEURON GPU + native are all enabled +(authoritative count: `ctest -N -R gpu` in your build tree). + +Native-only developers can rely on `_py_gpu_native` + `testneuron_gpu_*` unit +tests without running the full CoreNEURON GPU matrix. + +## Unit tests (`test/unit_tests/gpu/`) + +| Target | Exercises | +|--------|-----------| +| `testneuron_gpu_config` | `set_enable`, `set_backend`, guards | +| `testneuron_gpu_device_state` | Device lifecycle | +| `testneuron_gpu_upload` | Upload path | +| `testneuron_gpu_fadvance` | Timestep dispatch stubs | +| `testneuron_gpu_net_events` | Net event host stages | +| `testneuron_gpu_offload` | OpenACC offload helpers | +| `testneuron_gpu_device_assign` | MPI device assignment | + +Filter: `ctest -R testneuron_gpu` + +## Gap junction supplementary test + +Not part of the 19-ctest parity set; run manually after building with GPU: + +```console +cd test/gjtests && nrnivmodl +python3 test_par_gj_native_gpu.py +``` + +Compares `test_par_gj` dendrite voltages CPU vs `gpu.backend=native`. Ringtest +`-gap -gpu-native` (single rank) is an additional raster check; CTest +`neuron_gpu_native_mpi` does not pass `-gap` today. + +## Ringtest native benchmark + +External ringtest supports `-gpu-native` (orthogonal to CoreNEURON `-gpu`). +CTest: `external_ringtest::neuron_gpu_native_mpi`. See `gpu-testing.rst`. + +## CTest stability notes (Part A) + +- Pytest modtests need `LD_LIBRARY_PATH` including the build `lib` dir (NVHPC + `pgcudafat` loader). +- `libnrnmech` relinks when `libnrniv.so` changes to avoid stale ACC object + paths. + +Documented in Part A commit `b5fdf7a79`. \ No newline at end of file diff --git a/docs/dev/native-gpu-adoption/04-known-limitations.md b/docs/dev/native-gpu-adoption/04-known-limitations.md new file mode 100644 index 0000000000..463fa6368d --- /dev/null +++ b/docs/dev/native-gpu-adoption/04-known-limitations.md @@ -0,0 +1,70 @@ +# Known limitations (Phase B) + +Runtime limits documented in the Phase B contract. These are **not bugs to fix +in Phase B**; they define the boundary. + +## CVode / IDA + +Native GPU is fixed-step only. Enabling native GPU with CVode active raises an +error (`native_gpu_configuration_error()`). Use CPU NEURON or CoreNEURON for +variable-step models. + +## GPU spike priority queue + +Cross-rank spike scheduling remains on CPU with MPI. There is no device-resident +priority queue equivalent to the host NetCon queues. + +## Multi-thread OpenACC + +Validated default: `pc.nthread(1)`, `OMP_NUM_THREADS=1`. `pc.nthread(n>1)` +emits a one-time warning; per-thread CUDA contexts may fail on some platforms. + +## sparse13 / DAE models (extracellular, LinearMechanism) + +Native fixed-step is implemented as **separated GPU subphases**. For ODE tree +models, **GPU tree solve** (`solve_interleaved`) is on device. Extracellular and +`LinearMechanism` force `use_sparse13 = 1` and require **CPU sparse13 Gaussian +elimination** — the one subphase with no GPU implementation in Phase B. + +Other subphases (mechanism currents, `NET_RECEIVE`, post-solve in principle) are +not ruled out by the architecture, but Phase B does not certify DAE models on +`gpu.backend="native"`. `post_solve_needs_host_fallback()` often forces CPU +post-solve for DAE/LFP cases as well. + +## Gap junctions: CPU fixed-step fallback + +When `pc.setup_transfer()` is active (`nrnthread_v_transfer_` registered), native +GPU does **not** run `fadvance_gpu.cpp` for fixed-step integration. The CPU +`nrn_fixed_step_thread` body handles matrix setup, solve, post-solve, partrans +gather/MPI/scatter, and lastpart. Correctness is validated (e.g. +`test/gjtests/test_par_gj_native_gpu.py`); performance matches NEURON CPU (~1×), +not the GPU-accelerated non-gap path. + +Device-resident partrans gather/scatter — restoring GPU integration subphases while +keeping host/MPI only for cross-rank `MPI_Alltoallv` — is deferred (see +[05-future-work.md](05-future-work.md)). + +## LFP / extracellular post-solve fallback + +When `nrnthread_vi_compute_` is registered (extracellular `v+vext` sources for +partrans), `post_solve_needs_host_fallback()` forces host `nrn_update_voltage` +instead of `post_solve_on_device` on the **GPU integration path**. This applies +only when the native GPU step runs (no `nrnthread_v_transfer_` CPU fallback). + +## MPI-native modtest gap + +The 19-test parity set is single-process. MPI-native expansion (`spikes_mpi`, +`test_subworlds`, etc.) is not part of Phase B verification. + +## CMake: CoreNEURON required at configure + +`-DNRN_ENABLE_GPU=ON` without `-DNRN_ENABLE_CORENEURON=ON` fails configure. +This is a **build-system** limitation, not a runtime simulation limit. Native +simulations still avoid `coreneuron.enable=True`. Standalone GPU configure is +deferred (see [05-future-work.md](05-future-work.md)). + +## `coreneuron.enable` wins over `gpu.backend` + +If both `coreneuron.enable=True` and `gpu.backend="native"`, `pc.psolve` +embeds CoreNEURON — it does **not** run the native path. See the runtime table +in [`native-gpu-fixed-step.rst`](../native-gpu-fixed-step.rst). \ No newline at end of file diff --git a/docs/dev/native-gpu-adoption/05-future-work.md b/docs/dev/native-gpu-adoption/05-future-work.md new file mode 100644 index 0000000000..fcafca48ed --- /dev/null +++ b/docs/dev/native-gpu-adoption/05-future-work.md @@ -0,0 +1,34 @@ +# Future work (explicitly out of Phase B) + +Items below were considered during Phase B planning and **intentionally deferred** +to avoid mission creep. Phase B closes with PR7; these are not blockers for the +Phase B checklist. + +## Runtime features + +| Item | Notes | +|------|-------| +| CVode / IDA on native GPU | Separate project; different timestep contract | +| GPU spike priority queue | Requires MPI + device ordering design | +| Full multi-thread OpenACC | Harden per-thread CUDA contexts; remove warning | +| `sparse13` / extracellular / LFP on device | Eliminate per-step host fallbacks | +| Device partrans gather/scatter | Remove gap CPU fixed-step fallback; host/MPI only when `nrnmpi_numprocs > 1` | +| MPI-native modtest expansion | Beyond 19-test single-process parity | +| Per-step matrix host sync elimination | Performance optimization in `treeset.cpp` | + +## Build / install + +| Item | Notes | +|------|-------| +| `-DNRN_ENABLE_GPU=ON` without CoreNEURON | L0 decouple; OpenACC flags without `add_subdirectory(coreneuron)` | +| Native-only CI slice | Smaller `ctest` profile for minimal GPU installs | + +## Documentation / tooling (optional) + +| Item | Notes | +|------|-------| +| CodeTour / `@native-gpu` anchors | IDE-clickable map of L3–L7 | +| Compiler warnings cleanup | `src/neuron/gpu/` NVHPC pragma noise | +| Expanded C++ unit tests | Cover download flush, fallback matrix | + +Track as GitHub issues when opening Phase C or maintenance sprints. \ No newline at end of file diff --git a/docs/dev/native-gpu-adoption/PHASE-B-COMPLETE.md b/docs/dev/native-gpu-adoption/PHASE-B-COMPLETE.md new file mode 100644 index 0000000000..0a9bb48dbf --- /dev/null +++ b/docs/dev/native-gpu-adoption/PHASE-B-COMPLETE.md @@ -0,0 +1,57 @@ +# Phase B complete + +Checklist for closing native GPU **fixed-step** Phase B on branch +`hines-grok/feature/neuron-core-gpu-adoption`. + +## Runtime contract + +- [x] Fixed-step `pc.psolve` on `gpu.backend=native` without `coreneuron.enable` +- [x] OpenACC matrix solve + mechanism state + post-solve on device +- [x] CPU spike queues + MPI; GPU `NET_RECEIVE` and `net_send` buffer +- [x] Gap junctions: CPU fixed-step fallback when `setup_transfer` active (correctness; `test_par_gj_native_gpu`) +- [x] Batch download API (`download_flush_interval`) +- [x] CVode rejected when native GPU enabled +- [x] Threading warning for `pc.nthread(n>1)` + +## Verification + +- [x] 19/19 `*_py_gpu_native` modtests pass +- [x] `ctest -R gpu` stable on NVHPC dev build (Part A harness fixes) +- [x] Unit tests: `ctest -R testneuron_gpu` + +## Documentation + +- [x] PR6 — Sphinx scope contract (`native-gpu-fixed-step.rst`) +- [x] PR7 — This development journal (`native-gpu-adoption/*.md`) +- [x] Build/runtime tables explicit in Sphinx (CMake vs `gpu.*` / `coreneuron.enable`) + +## Code boundary PRs (Part A) + +| Commit | Summary | +|--------|---------| +| `ff48b5e78` | CVode rejection | +| `b5fdf7a79` | CTest / `libnrnmech` stability | +| `e800b0ffc` | Thread + spike/`NET_RECEIVE` policy docs | + +## Feature PRs (core path) + +| Commit | Summary | +|--------|---------| +| `b6bad25dd` … `e2eddbf30` | Matrix, mechanisms, post-solve, download, modtest parity | + +## Explicit deferrals (not Phase B) + +- Standalone `-DNRN_ENABLE_GPU=ON` configure (CoreNEURON=OFF) +- CVode on GPU, GPU spike queue, full multi-thread OpenACC +- PR4 warnings cleanup, PR5 unit-test expansion, CodeTour (optional) + +See [05-future-work.md](05-future-work.md). + +## Sign-off commands + +```console +ctest --output-on-failure -R '_py_gpu_native' +ctest --output-on-failure -R testneuron_gpu +``` + +Phase B journal last updated at commit cited in [appendix-commits.md](appendix-commits.md). \ No newline at end of file diff --git a/docs/dev/native-gpu-adoption/appendix-commits.md b/docs/dev/native-gpu-adoption/appendix-commits.md new file mode 100644 index 0000000000..2e1ea3d78a --- /dev/null +++ b/docs/dev/native-gpu-adoption/appendix-commits.md @@ -0,0 +1,237 @@ +# Appendix: commits + +Chronological log for `hines-grok/feature/neuron-core-gpu-adoption` (oldest first). +One paragraph per commit through PR7 (`2087fb131`). + +--- + +### `d53a3c17b` + +**Subject:** Add NRN_ENABLE_GPU and NRN_GPU_BACKEND CMake options (GPU adoption step 1) + +Introduces the user-facing `NRN_ENABLE_GPU` CMake option and `NRN_GPU_BACKEND=OpenACC` +scaffolding. Establishes L0 build switches separate from CoreNEURON's +`CORENRN_ENABLE_GPU`, with Phase A transition guard requiring CoreNEURON when +GPU is enabled. + +### `270919afa` + +**Subject:** GPU adoption step 1b: document CMAKE_CUDA_ARCHITECTURES in options.rst + +Documents optional `CMAKE_CUDA_ARCHITECTURES` for workstation GPU builds in +`docs/cmake_doc/options.rst`, reducing over-compilation when targeting a single +NVIDIA architecture. + +### `1ef4465f5` + +**Subject:** GPU adoption step 2: fix ringtest GPU MPI ctest NVHPC loader issue + +Fixes `external_ringtest::coreneuron_gpu_mpi` failures from NVHPC +`pgcudafat` loader issues by routing the MPI ringtest through `special-core` +(dump with `special`, simulate with `special-core --gpu`). + +### `bed18a476` + +**Subject:** GPU adoption step 3: propagate OpenACC compile+link to libnrniv + +Wires CoreNEURON-exported OpenACC compile and link flags into `libnrniv` and +entry points so NEURON proper can host OpenACC object code alongside classic CPU +sources. + +### `4d94ca7f0` + +**Subject:** GPU adoption step 4: add neuron::gpu offload scaffold + +Adds `src/neuron/gpu/` module with `neuron_gpu` static library and OpenACC +offload helpers — the foundation for L2–L4 native GPU code. + +### `92286d995` + +**Subject:** GPU adoption step 5: device_token lifecycle and NrnThread GPU fields + +Defines `device_token` and extends `NrnThread` with GPU bookkeeping fields so +upload state can be cached and invalidated with model structure changes. + +### `3101e4faf` + +**Subject:** GPU adoption step 6: default NMODL_NEURON_CODEGEN for GPU builds + +Sets default NMODL NEURON codegen for GPU builds so mechanism MOD files generate +OpenACC-capable C++ for the native path, not only CoreNEURON targets. + +### `847b56560` + +**Subject:** GPU adoption step 7: enable solve_interleaved GPU dispatch scaffolding + +Connects interleaved matrix solve dispatch to GPU when permutation is active — +prerequisite for L4 integration on device. + +### `688144b72` + +**Subject:** GPU adoption step 8: NMODL NEURON OpenACC codegen visitor + +Implements the NMODL visitor that emits NEURON-side OpenACC mechanism code, +parallel to CoreNEURON's GPU codegen path. + +### `233d5173d` + +**Subject:** Fix step 8 file headers: drop Blue Brain copyright, EOF newlines + +Housekeeping on step-8 NMODL sources: copyright headers and EOF newlines for +NEURON repo policy compliance. + +### `47ed2db60` + +**Subject:** GPU adoption step 9: SOA and InterleaveInfo device upload (MVP) + +Minimum viable upload of SOA layout and `InterleaveInfo` to device — first +end-to-end data present for GPU timestep experiments. + +### `7a1dc70fb` + +**Subject:** GPU adoption step 10: native psolve dispatch scaffolding + +Routes `nrn_fixed_step_thread` to `fadvance_gpu` when native backend is enabled +(L3 dispatch scaffolding). + +### `95c28200f` + +**Subject:** GPU adoption step 11: host-side network event stages (Phase B1) + +Adds host-side network event delivery stages before/after the GPU integration +step, preserving CPU spike queue semantics during native GPU steps. + +### `3a6955d05` + +**Subject:** GPU adoption step 12: end-to-end native psolve scaffolding (G4 subset) + +First end-to-end native `pc.psolve` for a G4 modtest subset — proves the +scaffolded path can complete a simulation. + +### `0ba3acef1` + +**Subject:** GPU adoption step 13: ringtest native GPU benchmark harness + +Adds `-gpu-native` ringtest support and CTest `neuron_gpu_native_mpi` for +benchmarking the native path against reference spike files. + +### `0100ec7d9` + +**Subject:** GPU adoption step 14: MPI multi-GPU device assignment + +Implements `device_id = mpi_local_rank % num_gpus_per_node` policy for native +GPU, matching CoreNEURON `init_gpu()` behavior. + +### `94e689da3` + +**Subject:** GPU adoption step 15: Python GPU API consolidation + MKL guard + +Introduces `neuron.gpu` module with HOC sync; deprecates `coreneuron.gpu` shims. +Adds Intel OpenMP/MKL import guard for NVHPC OpenACC compatibility. + +### `2663a5c50` + +**Subject:** GPU adoption step 16: test stabilization (ringtest spk2, backend_helper copies) + +Stabilizes ringtest spike comparisons and `backend_helper.py` state copies between +native and CoreNEURON test paths. + +### `0dd2d055b` + +**Subject:** GPU adoption PR 7b: OpenACC setup_tree_matrix in treeset.cpp + +OpenACC offload of `setup_tree_matrix` for the native path — moves axial matrix +assembly work toward device execution. + +### `a225a10f0` + +**Subject:** GPU adoption PR 11b+16: net_send GPU buffer + gap/VecPlay sync + +Adds `NetSendBuffer` for device-generated `net_send` events and gap junction / +VecPlay device↔host sync hooks (L6 communication). + +### `a6c488e0f` + +**Subject:** Fix GPU test failures: upload parent index, hybrid host/device sync + +Fixes parent index upload and hybrid sync bugs uncovered by expanding modtests — +correctness fixes before parity expansion. + +### `8795cd51e` + +**Subject:** Fix native GPU fast_imem sync for pointer modtests + +Ensures `fast_imem` device state is visible where pointer MOD mechanisms read +it — unblocks pointer-related native GPU tests. + +### `41ce00183` + +**Subject:** Enable GPU axial matrix assembly and GPU solve_interleaved on native path + +Completes L4 integration: GPU axial matrix assembly and interleaved solve on the +native fixed-step path. + +### `b6bad25dd` + +**Subject:** Run GPU post-solve voltage update and fast_imem on device + +Moves post-solve voltage update, `fast_imem`, and capacity current to device +(`post_solve_on_device`), reducing per-step host pulls. + +### `574b22a2f` + +**Subject:** Upload padded Memb_list shells for native GPU mechanism path + +Uploads padded `Memb_list` layouts for OpenACC mechanism kernels — device SOA +parity with CoreNEURON assumptions. + +### `e0037c448` + +**Subject:** Add batch GPU download API for Vector.record and graph flush + +Adds `download_flush_interval` and batch host download for recordings and HOC +graph reads (L7). + +### `e2eddbf30` + +**Subject:** Expand G4 native GPU modtests to full _py_gpu parity (PR 19) + +Expands native modtests to **19** `*_py_gpu_native` targets mirroring +single-process `*_py_gpu` — Phase B L8 verification anchor. + +### `ff48b5e78` + +**Subject:** Reject native GPU when CVode is active (Phase B boundary) + +Adds `native_gpu_configuration_error()` at Python, HOC, and `nrn_fixed_step` +entry — declares CVode out of scope for native GPU. + +### `b5fdf7a79` + +**Subject:** Stabilize GPU ctest harness for pytest modtests (Phase B) + +Sets `LD_LIBRARY_PATH` for pytest modtests; relinks `libnrnmech` when +`libnrniv.so` changes; updates `gpu-testing.rst` test counts. + +### `e800b0ffc` + +**Subject:** Document native GPU threading and spike/NET_RECEIVE policy (Phase B) + +One-time `pc.nthread(n>1)` warning; documents spike/`NET_RECEIVE` split in +`gpu-testing.rst`, `net_events.hpp`, `net_send_buffer.hpp`. + +### `8fe50fa88` + +**Subject:** Add native GPU fixed-step Sphinx scope contract (PR6 / Phase B) + +Adds `docs/dev/native-gpu-fixed-step.rst` as canonical Phase B reference with +L0–L8 layers, build/runtime tables, mermaid timestep flow, and API docs. +Dedupes scope text from `gpu-testing.rst`; cross-links CMake options. + +### `2087fb131` + +**Subject:** Add native GPU adoption dev journal and Phase B closure (PR7) + +Adds `docs/dev/native-gpu-adoption/` development journal (overview through +commit appendix, `PHASE-B-COMPLETE` checklist). Links from dev index and Sphinx +scope page. Closes Phase B documentation scope without L0 build decouple work. \ No newline at end of file diff --git a/docs/dev/native-gpu-fixed-step.rst b/docs/dev/native-gpu-fixed-step.rst new file mode 100644 index 0000000000..33c622c2cf --- /dev/null +++ b/docs/dev/native-gpu-fixed-step.rst @@ -0,0 +1,407 @@ +.. _native-gpu-fixed-step: + +Native GPU fixed-step (Phase B) +############################### + +This page is the **scope contract** for NEURON's native GPU backend +(``gpu.backend="native"``). It describes what Phase B supports, what is +explicitly excluded, how a fixed-step timestep is organized, and how to +configure runtime options. + +For CTest commands, ringtest benchmarks, and CI troubleshooting, see +:doc:`gpu-testing`. + +Overview +******** + +When NEURON is built with ``-DNRN_ENABLE_GPU=ON`` (NVHPC + OpenACC), models +can run fixed-step ``pc.psolve`` on the GPU **without enabling CoreNEURON at +runtime** (``coreneuron.enable=False``). That is different from the CMake +default ``NRN_ENABLE_CORENEURON=OFF`` — see :ref:`native-gpu-build-runtime`. + +.. code-block:: python + + from neuron import h, gpu + + pc = h.ParallelContext() + gpu.enable = True + gpu.backend = "native" + pc.psolve(100) + +Phase B delivers CoreNEURON-parity integration for single-process modtests +(19 ``*_py_gpu_native`` ctests) while keeping cross-rank spike scheduling on +the CPU — the same split CoreNEURON uses. + +Phase B contract +**************** + +**In scope** + +- Fixed-step ``pc.psolve`` and ``h.fadvance`` on the native path +- OpenACC matrix solve (interleaved when permuted) +- Mechanism state on device (padded ``Memb_list`` upload) +- Post-solve on device: voltage update, fast_imem, capacity current +- ``NET_RECEIVE`` mechanism computation on GPU during the step +- CPU spike priority queues + MPI exchange at the minimum NetCon delay interval +- GPU ``net_send`` buffering; cross-cell events to CPU/MPI; self-events may flush on device +- Gap junctions: **correctness** on native GPU when ``pc.setup_transfer()`` is + active; fixed-step integration runs the **CPU body** (full fallback) until + device partrans gather/scatter is implemented +- Batch host download API for ``Vector.record`` and graphs (``download_flush_interval``) + +**Out of scope (deferred)** + +- CVode / IDA on native GPU (enabling native GPU with CVode active raises an error) +- GPU spike priority queue +- Full multi-thread OpenACC maturity (``pc.nthread(1)`` is the validated default) +- GPU sparse13 / DAE Gaussian elimination (extracellular, ``LinearMechanism``) +- LFP post-solve hooks on device (host fallback where registered) +- MPI-native modtest expansion beyond current parity set + +Supported and unsupported matrix +******************************** + ++-------------------------------+------------------+---------------------------+ +| Feature | Native GPU | Notes | ++===============================+==================+===========================+ +| Fixed-step ``pc.psolve`` | Supported | Primary Phase B path | ++-------------------------------+------------------+---------------------------+ +| ``h.fadvance`` | Supported | Same dispatch as psolve | ++-------------------------------+------------------+---------------------------+ +| CVode / IDA | **Unsupported** | Use CPU or CoreNEURON | ++-------------------------------+------------------+---------------------------+ +| Mechanisms (NMODL) | Supported | OpenACC codegen path | ++-------------------------------+------------------+---------------------------+ +| ``NET_RECEIVE`` | Supported | Computed on GPU | ++-------------------------------+------------------+---------------------------+ +| Cross-rank NetCon spikes | CPU queues + MPI | Sparse at min delay | ++-------------------------------+------------------+---------------------------+ +| ``net_send`` self-events | GPU buffer | May flush on device | ++-------------------------------+------------------+---------------------------+ +| Gap junctions | Supported | CPU fixed-step fallback | ++-------------------------------+------------------+---------------------------+ +| ``Vector.record`` | Supported | Batch download API | ++-------------------------------+------------------+---------------------------+ +| HOC graphs | Supported | Flush pulls GPU state | ++-------------------------------+------------------+---------------------------+ +| ``pc.nthread(n>1)`` | Experimental | One-time warning emitted | ++-------------------------------+------------------+---------------------------+ +| Extracellular | **Unsupported** | Needs CPU ``sparse13`` solve| ++-------------------------------+------------------+---------------------------+ +| ``LinearMechanism`` | **Unsupported** | Needs CPU ``sparse13`` solve| ++-------------------------------+------------------+---------------------------+ +| Tree matrix solve (ODE) | Supported | GPU ``solve_interleaved`` | ++-------------------------------+------------------+---------------------------+ +| ``sparse13`` Gaussian elim. | **Unsupported** | CPU only; future phase | ++-------------------------------+------------------+---------------------------+ +| LFP / ``nrnthread_vi_compute_`` | Host fallback | Post-solve on host only | ++-------------------------------+------------------+---------------------------+ +| CoreNEURON embedded backend | Separate path | ``gpu.backend=coreneuron``| ++-------------------------------+------------------+---------------------------+ + +Conceptual layers +***************** + +Use these levels when reading code, tests, or the development journal (PR7): + ++-------+--------------------+-----------------------------------------------+ +| Level | Name | Key artifacts | ++=======+====================+===============================================+ +| L0 | Build | ``NRN_ENABLE_GPU``, ``NRN_ENABLE_CORENEURON``, NVHPC | ++-------+--------------------+-----------------------------------------------+ +| L1 | Runtime API | :mod:`neuron.gpu`, ``ParallelContext`` GPU | ++-------+--------------------+-----------------------------------------------+ +| L2 | Device lifecycle | ``device_state``, ``upload`` / teardown | ++-------+--------------------+-----------------------------------------------+ +| L3 | Timestep dispatch | ``fadvance.cpp`` → ``fadvance_gpu.cpp`` | ++-------+--------------------+-----------------------------------------------+ +| L4 | Integration step | Matrix setup, solve, post-solve | ++-------+--------------------+-----------------------------------------------+ +| L5 | Host fallbacks | ``post_solve_needs_host_fallback()`` | ++-------+--------------------+-----------------------------------------------+ +| L6 | Communication | Spike queues (CPU), gap MPI, ``NetSendBuffer``| ++-------+--------------------+-----------------------------------------------+ +| L7 | Recording | ``download.cpp``, batch flush intervals | ++-------+--------------------+-----------------------------------------------+ +| L8 | Verification | ``*_py_gpu_native`` modtests, unit tests | ++-------+--------------------+-----------------------------------------------+ + +Fixed-step subphases (native GPU) +********************************* + +Native fixed-step integration is split into **subphases** (``fadvance_gpu.cpp``, +``post_solve.cpp``, ``net_events.cpp``, …). Most subphases can run on the GPU; +the notable exception for DAE models is the **linear solve** — tree ODE models use +GPU ``solve_interleaved``, while extracellular / ``LinearMechanism`` require CPU +``sparse13`` Gaussian elimination (``spFactor`` / ``spSolve``). That separation +is architectural: a future GPU sparse solver is its own phase (like CVode-on-GPU), +not a rewrite of the whole native path. + ++---------------------------+--------------------+-------------------------------+ +| Subphase | ODE tree (Phase B) | DAE (extracellular / linmod) | ++===========================+====================+===============================+ +| Spike queues + MPI | CPU | CPU | ++---------------------------+--------------------+-------------------------------+ +| ``deliver_net_events`` | CPU | CPU | ++---------------------------+--------------------+-------------------------------+ +| ``setup_tree_matrix`` + | GPU | GPU-capable in principle; | +| NMODL mechanism currents | | sparse13 assembly differs | ++---------------------------+--------------------+-------------------------------+ +| Matrix solve | **GPU tree solve** | **CPU sparse13 only** | ++---------------------------+--------------------+-------------------------------+ +| Post-solve (V, fast_imem) | GPU | Host fallback typical | ++---------------------------+--------------------+-------------------------------+ +| ``NET_RECEIVE`` / nonvint | GPU | GPU-capable in principle | +| (lastpart) | | | ++---------------------------+--------------------+-------------------------------+ +| ``net_send`` buffer | GPU | GPU-capable in principle | ++---------------------------+--------------------+-------------------------------+ +| Gap junction transfer | **CPU fixed-step | Same (if gaps present) | +| (``setup_transfer``) | fallback** (below) | | ++---------------------------+--------------------+-------------------------------+ +| Recording flush | GPU → host batch | Same API | ++---------------------------+--------------------+-------------------------------+ + +Phase B **validates the ODE column** (19 ``*_py_gpu_native`` modtests). DAE rows +are not production-supported on ``gpu.backend="native"`` today; performance of +mixed CPU-solve / GPU-subphase paths is untested. + +One fixed step — ODE tree (diagram) +*********************************** + +Diagram below assumes **``gpu.enable=True``**, **``gpu.backend="native"``**, and +``coreneuron.enable=False`` (classic ``pc.psolve`` on the native path). Scope: +**fixed-step, model type 1 (ODE tree)**, no extracellular or +``LinearMechanism``. Variable-step (CVode) and multi-thread OpenACC are separate +limitations (see matrix above). + +.. mermaid:: + + flowchart TD + S{nrnthread_v_transfer_ registered?} + S -->|yes| CPUALL[CPU fixed-step: deliver, matrix, solve, update, gap transfer, lastpart] + S -->|no| A[CPU: deliver_net_events] + A --> B[GPU: setup_tree_matrix + NMODL currents] + B --> C[GPU: tree solve solve_interleaved] + C --> D{post_solve host fallback?} + D -->|no| E[GPU: post_solve V, fast_imem, capacity] + D -->|yes| F[CPU: post_solve nrn_update_voltage] + E --> I[GPU: lastpart NET_RECEIVE + nonvint + vecplay] + F --> I + I --> J{download flush interval?} + J -->|yes| K[batch_download_to_host] + J -->|no| L[defer to next flush / psolve end] + K --> M[advance step counter] + L --> M + CPUALL --> M + +On this path ``nt.compute_gpu`` is set for integration. NMODL ``BREAKPOINT`` +currents (``nrn_cur_*`` OpenACC) and OpenACC axial assembly in ``setup_tree_matrix`` +run on the GPU. **GPU tree solve** is ``solve_interleaved`` / ``solve_interleaved1|2`` +(OpenACC in ``cellorder*.cpp``) — the same tridiagonal structure as classic +``triang``/``bksub``, not a general sparse solver. + +``NET_RECEIVE`` mechanism computation and related nonvint work run on the GPU +during **lastpart** (after the solve/post-solve block). Cross-rank spike +**scheduling** remains on CPU (see spike policy in :doc:`gpu-testing`). + +**Gap junction transfer** (``ParallelContext.setup_transfer``) registers +``nrnthread_v_transfer_`` whenever partrans targets exist — including +single-process, single-thread models with gaps (not only when MPI or +``pc.nthread(n)>1``). + +Phase B native GPU **does not** run the hybrid GPU integration path above when +``nrnthread_v_transfer_`` is set. ``nrn_fixed_step_thread`` in ``fadvance.cpp`` +dispatches the **full CPU fixed-step body** instead of ``fadvance_gpu.cpp`` until +device-resident partrans gather/scatter is complete. ``gpu.enable=True`` still +uploads mechanisms and state to device, but matrix setup, solve, post-solve, gap +transfer, and lastpart execute on the CPU for correctness (validated by +``test/gjtests/test_par_gj_native_gpu.py`` and ringtest ``-gap``). Runtime is +therefore ~1× NEURON CPU for gap models, not the slower hybrid path seen on +non-gap workloads. + +On that CPU path, partrans uses three phases (``partrans.cpp``): + +1. **Gather** source values into transfer buffers (``mpi_transfer``: ``outsrc_buf_[i] + = *poutsrc_[i]``). +2. **MPI exchange** if ``nrnmpi_numprocs > 1`` (``MPI_Alltoallv`` or sparse variant + into ``insrc_buf_``). Skipped on one rank. +3. **Scatter** to targets (``thread_transfer`` in ``nonvint``, per thread: + ``*(ttd.tv[i]) = *(ttd.sv[i])``). + +A future optimization would restore the GPU integration subphases and move phases +1 and 3 to device buffers, touching host/MPI only when ``nrnmpi_numprocs > 1``. +The hybrid sync helpers in ``sync.cpp`` (``sync_gap_after_voltage_update``, +``sync_gap_after_host_voltage_update``) remain for that path; they are not used on +the Phase B CPU fallback dispatch. + +For DAE models, only the **matrix solve** subphase must swap to CPU ``sparse13``; +other subphases are not inherently excluded by the subphase design, but Phase B +does not certify extracellular or ``LinearMechanism`` on the native path. + +**Post-solve host fallback** (``post_solve_needs_host_fallback()``) forces CPU +``nrn_update_voltage`` when LFP/partrans registers ``nrnthread_vi_compute_``, +or when ``use_sparse13`` / extracellular is active — independent of the tree-solve +vs sparse13 distinction for ODE models. + +Spike exchange with MPI occurs at the **minimum NetCon delay** cadence (often +many fixed steps), not every ``dt``. + +Runtime configuration +********************* + +Primary API: :mod:`neuron.gpu` (HOC equivalents on ``ParallelContext``). + +.. code-block:: python + + from neuron import gpu + + gpu.enable = True + gpu.backend = "native" # fixed-step native path + gpu.permute = 2 # default when enable=True + gpu.device_count = 0 # 0 = all GPUs on the node + gpu.download_flush_interval = 1 # steps between host pulls (0 = psolve end only) + +Context manager (mirrors ``coreneuron()``): + +.. code-block:: python + + with gpu(enable=True, backend="native", download_flush_interval=10): + pc.psolve(tstop) + +Build-time vs runtime configuration +*********************************** + +.. _native-gpu-build-runtime: + +The old single truth table mixed **CMake build options** (fixed at configure +time) with **Python/HOC runtime switches** (chosen per simulation). They are +independent layers. + +Build-time (CMake) +------------------ + +These options decide which code is compiled into your ``special`` binary. They +are **not** implied by ``gpu.backend`` or ``coreneuron.enable``. + ++-------------------------------+--------------------------------+--------------------------------+ +| ``NRN_ENABLE_GPU`` | ``NRN_ENABLE_CORENEURON`` | Configure result | ++===============================+================================+================================+ +| OFF (default) | OFF (default) | CPU NEURON only; ``gpu.*`` | +| | | has no effect at runtime | ++-------------------------------+--------------------------------+--------------------------------+ +| OFF | ON | CoreNEURON embedded path only | +| | | (no native GPU dispatch) | ++-------------------------------+--------------------------------+--------------------------------+ +| ON | OFF | **Configure fails** (Phase A–B | +| | | transition; CMake still | +| | | requires CoreNEURON) | ++-------------------------------+--------------------------------+--------------------------------+ +| ON | ON | Both paths available at | +| | | runtime (typical dev/CI build) | ++-------------------------------+--------------------------------+--------------------------------+ + +Most users leave ``NRN_ENABLE_CORENEURON`` at its default **OFF**. Today you +cannot combine that default with ``NRN_ENABLE_GPU=ON``: you must pass +``-DNRN_ENABLE_CORENEURON=ON`` to configure a native-GPU-capable build. That +builds the CoreNEURON library into the install tree, but you still choose at +**runtime** whether ``pc.psolve`` embeds CoreNEURON or runs the native path. + +Runtime dispatch (``pc.psolve``) +-------------------------------- + +``gpu.backend`` and ``coreneuron.enable`` answer different questions: + +- ``coreneuron.enable`` — should ``pc.psolve`` **transfer the model and run + inside CoreNEURON**? +- ``gpu.enable`` + ``gpu.backend`` — on the **classic NEURON** path, should + fixed-step integration run on the **native GPU** backend? + +``pc.psolve`` checks ``coreneuron.enable`` **first**: + +1. ``coreneuron.enable=True`` → CoreNEURON embedded (``nrncore_psolve``). + GPU execution inside CoreNEURON is controlled by ``coreneuron.gpu`` (legacy; + sets ``gpu.enable`` and ``gpu.backend="coreneuron"``), not by + ``gpu.backend="native"``. +2. ``coreneuron.enable=False`` → classic NEURON (``netpar_solve``). Native + GPU runs only when ``gpu.enable=True`` **and** ``gpu.backend="native"``. + +Requires ``NRN_ENABLE_GPU=ON`` at build time; otherwise ``gpu.enable`` is a +no-op and every row below is CPU NEURON. + ++---------------------+--------------+----------------+---------------------------+ +| coreneuron.enable | gpu.enable | gpu.backend | Effective ``pc.psolve`` | ++=====================+==============+================+===========================+ +| False | False | * | CPU NEURON | ++---------------------+--------------+----------------+---------------------------+ +| False | True | ``native`` | **Native GPU** fixed-step | ++---------------------+--------------+----------------+---------------------------+ +| False | True | ``coreneuron`` | CPU NEURON (backend has | +| | | | no effect without embed) | ++---------------------+--------------+----------------+---------------------------+ +| True | False | * | CoreNEURON embedded, CPU | ++---------------------+--------------+----------------+---------------------------+ +| True | True | ``coreneuron`` | CoreNEURON embedded, GPU | +| | | | (``coreneuron.gpu`` / | +| | | | ``--gpu`` launcher arg) | ++---------------------+--------------+----------------+---------------------------+ +| True | True | ``native`` | CoreNEURON embedded | +| | | | (**not** native GPU; | +| | | | ``coreneuron.enable`` | +| | | | wins over ``gpu.backend``)| ++---------------------+--------------+----------------+---------------------------+ + +**Phase B native recipe** (after a GPU-capable build): + +.. code-block:: python + + from neuron import h, gpu + + # coreneuron.enable stays False (default) + gpu.enable = True + gpu.backend = "native" + pc.psolve(tstop) + +Download flush interval +*********************** + +Recording and HOC reads of GPU-resident state use a **batch download** API. +By default (``download_flush_interval=1``), post-solve voltages and fast_imem +are pulled to the host every step. Set ``0`` to defer all pulls until +``psolve`` ends (best throughput; recordings see data only at flush boundaries). + +HOC: ``pc.gpu_download_flush_interval(interval)`` + +Threading +********* + +Native GPU modtests run with ``OMP_NUM_THREADS=1`` and ``pc.nthread(1)``. +Using ``pc.nthread(n)`` for ``n > 1`` emits a one-time warning: per-thread +OpenACC CUDA contexts are not fully validated on all platforms. + +Testing +******* + +Quick parity check after building with GPU tests enabled: + +.. code-block:: console + + ctest -N -R '_py_gpu_native' + ctest --output-on-failure -R '_py_gpu_native' + +Broader GPU regression: + +.. code-block:: console + + ctest --output-on-failure -R gpu + +See :doc:`gpu-testing` for ringtest, MPI device assignment, and NVHPC linker +notes. + +Related documentation +********************* + +- :doc:`/dev/native-gpu-adoption/00-overview` — Phase B development journal (design, commits) +- :ref:`cmake-nrn-enable-gpu-option` — CMake configure +- :doc:`/nmodl/gpu_codegen` — mechanism OpenACC codegen +- :doc:`gpu-testing` — CTest and benchmark workflows \ No newline at end of file diff --git a/docs/nmodl/gpu_codegen.rst b/docs/nmodl/gpu_codegen.rst new file mode 100644 index 0000000000..00df700950 --- /dev/null +++ b/docs/nmodl/gpu_codegen.rst @@ -0,0 +1,64 @@ +NMODL codegen for GPU-enabled NEURON builds +=========================================== + +When NEURON is configured with ``NRN_ENABLE_GPU=ON``, the CMake API +``create_nrnmech`` (and the installed ``nrnivmodl`` CMake wrapper) default to +the **NMODL** transpiler with ``--neuron`` (option ``NMODL_NEURON_CODEGEN``). +The legacy **NOCMODL** path cannot emit OpenACC code and is rejected when +``create_nrnmech`` is invoked without ``NMODL_NEURON_CODEGEN`` on GPU builds. + +The shell ``nrnivmodl`` makefile workflow still uses NOCMODL during the Phase A +transition so existing GPU ctests keep passing; use ``create_nrnmech`` with +``NMODL_NEURON_CODEGEN`` (or ``NRN_ENABLE_NMODL=ON``) for new GPU-targeted +mechanism work. + +``create_nrnmech`` / CMake workflow +----------------------------------- + +.. code-block:: cmake + + create_nrnmech( + NEURON + NMODL_NEURON_CODEGEN # implied when NRN_ENABLE_GPU=ON + MOD_FILES hh.mod) + +``nrnivmodl`` shell workflow +---------------------------- + +.. code-block:: bash + + nrnivmodl mod # NOCMODL (Phase A transition; see above) + nrnivmodl -coreneuron mod # CoreNEURON mechanisms via NMODL + OpenACC + nrnivmodl -nmodl $(which nmodl) mod # explicit NMODL NEURON codegen + +Passing ``-nocmodl`` or pointing ``-nmodl`` at a ``nocmodl`` binary fails on +GPU-enabled builds. + +NMODL vs NOCMODL feature gaps (NEURON) +-------------------------------------- + +Use NMODL for new GPU-targeted mechanisms. Known construct coverage differences +that affect porting legacy MOD files: + ++------------------+----------------------------+----------------------------------+ +| MOD construct | NOCMODL (legacy) | NMODL ``--neuron`` | ++==================+============================+==================================+ +| KINETIC | Supported | Supported; preferred for new code| ++------------------+----------------------------+----------------------------------+ +| TABLE | Supported | Supported; verify table ranges | ++------------------+----------------------------+----------------------------------+ +| POINTER | Supported | Supported with handle migration | ++------------------+----------------------------+----------------------------------+ +| NET_RECEIVE | Supported | Supported; GPU delivery host-side| +| | | in Phase B native GPU | ++------------------+----------------------------+----------------------------------+ +| SOLVE methods | Full legacy set | Subset; check solver compatibility| ++------------------+----------------------------+----------------------------------+ + +When ``NRN_ENABLE_GPU=ON`` and a CUDA compiler is available, +``create_nrnmech`` appends ``acc --oacc`` to ``NMODL_NEURON_EXTRA_ARGS`` so +NMODL emits OpenACC via ``CodegenNeuronAccVisitor`` (``nrn_pragma_acc`` on +mechanism entrypoints). Review ``hh.mod`` ACC output before enabling native GPU +psolve (PR 12). + +See also :doc:`transpiler/readme` and :doc:`language/nmodl_neuron_extension`. \ No newline at end of file diff --git a/docs/nmodl/language.rst b/docs/nmodl/language.rst index 38358fae40..c83a656e22 100644 --- a/docs/nmodl/language.rst +++ b/docs/nmodl/language.rst @@ -9,3 +9,4 @@ This section describes the Neuron Model Description Language and how to use it t language/nmodl language/nmodl_neuron_extension + gpu_codegen diff --git a/external/coding-conventions b/external/coding-conventions index f1915f5d4f..5873afc8ea 160000 --- a/external/coding-conventions +++ b/external/coding-conventions @@ -1 +1 @@ -Subproject commit f1915f5d4f169569de9144c13d58afe20a1ea73e +Subproject commit 5873afc8ead90e1aed7884e841a16865d1da3769 diff --git a/share/lib/python/neuron/CMakeLists.txt b/share/lib/python/neuron/CMakeLists.txt index 30da70e227..5abc035d50 100644 --- a/share/lib/python/neuron/CMakeLists.txt +++ b/share/lib/python/neuron/CMakeLists.txt @@ -32,6 +32,7 @@ set(NRN_PYTHON_FILES_LIST tests/test_vector.py __init__.py coreneuron.py + gpu.py doc.py gui2/setup_threejs.py gui2/config.py diff --git a/share/lib/python/neuron/__init__.py b/share/lib/python/neuron/__init__.py index b21e4cbc9b..c6d49aa517 100644 --- a/share/lib/python/neuron/__init__.py +++ b/share/lib/python/neuron/__init__.py @@ -236,12 +236,14 @@ def _check_for_intel_openmp() -> None: import ctypes from neuron.config import arguments - # These checks are only relevant for shared library builds with CoreNEURON GPU support enabled. - if ( - not arguments["NRN_ENABLE_CORENEURON"] - or not arguments["CORENRN_ENABLE_GPU"] - or not arguments["CORENRN_ENABLE_SHARED"] - ): + # NVHPC OpenACC and Intel OpenMP conflict when both are loaded in-process. + # Trigger for native NEURON GPU builds (NRN_ENABLE_GPU) or shared CoreNEURON GPU libs. + needs_check = arguments.get("NRN_ENABLE_GPU") or ( + arguments["NRN_ENABLE_CORENEURON"] + and arguments["CORENRN_ENABLE_GPU"] + and arguments["CORENRN_ENABLE_SHARED"] + ) + if not needs_check: return current_exe = ctypes.CDLL(None) @@ -275,6 +277,11 @@ def _check_for_intel_openmp() -> None: _check_for_intel_openmp() +try: + from . import gpu +except ImportError: + pass + _original_hoc_file = None if not hasattr(hoc, "__file__"): # first try is to derive from neuron.__file__ diff --git a/share/lib/python/neuron/coreneuron.py b/share/lib/python/neuron/coreneuron.py index 4557284506..5354cdda81 100644 --- a/share/lib/python/neuron/coreneuron.py +++ b/share/lib/python/neuron/coreneuron.py @@ -1,4 +1,5 @@ import sys +import warnings class CoreNEURONContextHelper(object): @@ -67,6 +68,25 @@ class coreneuron(object): False """ + def _warn_deprecated(self, old_name, new_name): + warnings.warn( + "coreneuron.{} is deprecated; use {} instead".format(old_name, new_name), + DeprecationWarning, + stacklevel=3, + ) + + def _sync_to_gpu_module(self): + try: + from neuron import gpu as ngpu + except ImportError: + return + ngpu.enable = self._gpu + if self._gpu: + ngpu.backend = "coreneuron" + ngpu.device_count = self._num_gpus + if self._cell_permute is not None: + ngpu.permute = self._cell_permute + def __init__(self): self._enable = False self._gpu = False @@ -119,7 +139,9 @@ def gpu(self): @gpu.setter def gpu(self, value): + self._warn_deprecated("gpu", "gpu.enable (and gpu.backend for native)") self._gpu = bool(int(value)) + self._sync_to_gpu_module() # If cell_permute has been set to a value incompatible with the new GPU # setting, change it and print a warning. if ( @@ -142,7 +164,9 @@ def num_gpus(self): @num_gpus.setter def num_gpus(self, value): + self._warn_deprecated("num_gpus", "gpu.device_count") self._num_gpus = int(value) + self._sync_to_gpu_module() @property def file_mode(self): @@ -166,9 +190,11 @@ def cell_permute(self): @cell_permute.setter def cell_permute(self, value): + self._warn_deprecated("cell_permute", "gpu.permute") value = int(value) assert value in self.valid_cell_permute() self._cell_permute = value + self._sync_to_gpu_module() @property def warp_balance(self): diff --git a/share/lib/python/neuron/gpu.py b/share/lib/python/neuron/gpu.py new file mode 100644 index 0000000000..ea7ac594cc --- /dev/null +++ b/share/lib/python/neuron/gpu.py @@ -0,0 +1,142 @@ +import sys + + +class GPUContextHelper(object): + def __init__(self, gpu_module, new_values): + self._gpu = gpu_module + self._new_values = new_values + self._old_values = None + + def __enter__(self): + assert self._new_values is not None + assert self._old_values is None + self._old_values = {} + for key, value in self._new_values.items(): + self._old_values[key] = getattr(self._gpu, key) + setattr(self._gpu, key, value) + + def __exit__(self, exc_type, exc_val, exc_tb): + assert self._new_values is not None + assert self._old_values is not None + for key in reversed(self._new_values.keys()): + setattr(self._gpu, key, self._old_values[key]) + return False + + +class gpu(object): + """Runtime configuration for NEURON GPU execution.""" + + _VALID_BACKENDS = frozenset({"native", "coreneuron"}) + + def __init__(self): + self._enable = False + self._backend = "coreneuron" + self._device_count = 0 + self._permute = None + self._download_flush_interval = 1 + + def __call__(self, **kwargs): + return GPUContextHelper(self, kwargs) + + def _pc(self): + from neuron import h + + return h.ParallelContext() + + @property + def enable(self): + return self._enable + + @enable.setter + def enable(self, value): + new_enable = bool(int(value)) + was_enabled = self._enable + if new_enable and self._backend == "native": + self._validate_native_fixed_step() + self._enable = new_enable + self._sync_to_hoc() + if new_enable and not was_enabled: + self._apply_default_permute() + + @property + def backend(self): + return self._backend + + @backend.setter + def backend(self, value): + backend = str(value).lower() + if backend not in self._VALID_BACKENDS: + raise ValueError( + "gpu.backend must be 'native' or 'coreneuron', got {!r}".format(value) + ) + if backend == "native" and self._enable: + self._validate_native_fixed_step() + self._backend = backend + self._sync_to_hoc() + + @property + def device_count(self): + """Number of GPUs per node (0 = all available).""" + return self._device_count + + @device_count.setter + def device_count(self, value): + self._device_count = int(value) + self._sync_to_hoc() + + @property + def download_flush_interval(self): + """Steps between host downloads during psolve (0 = end of psolve only).""" + return self._download_flush_interval + + @download_flush_interval.setter + def download_flush_interval(self, value): + self._download_flush_interval = int(value) + self._sync_to_hoc() + + @property + def permute(self): + """Cell permutation order (0, 1, or 2). Default 2 when enable is True.""" + if self._permute is None: + return 2 if self._enable else 0 + return self._permute + + @permute.setter + def permute(self, value): + value = int(value) + if value not in {0, 1, 2}: + raise ValueError("gpu.permute must be 0, 1, or 2, got {!r}".format(value)) + self._permute = value + if self._enable: + self._pc().optimize_node_order(value) + + def _apply_default_permute(self): + if self._permute is None: + self._pc().optimize_node_order(2) + else: + self._pc().optimize_node_order(self._permute) + + def _validate_native_fixed_step(self): + from neuron import h + + if h.CVode().active(): + raise RuntimeError( + "native GPU backend (gpu.backend='native') requires fixed-step integration; " + "deactivate CVode before enabling native GPU" + ) + + def _sync_to_hoc(self): + try: + pc = self._pc() + except Exception: + return + # HOC gpu_* methods are registered only when NRN_ENABLE_GPU is set at build time. + if not hasattr(pc, "gpu_enable"): + return + pc.gpu_enable(int(self._enable)) + pc.gpu_backend(self._backend) + pc.gpu_device_count(int(self._device_count)) + pc.gpu_download_flush_interval(int(self._download_flush_interval)) + + +sys.modules[__name__] = gpu() diff --git a/src/coreneuron/permute/cellorder.cpp b/src/coreneuron/permute/cellorder.cpp index f708c862cc..809769b2a9 100644 --- a/src/coreneuron/permute/cellorder.cpp +++ b/src/coreneuron/permute/cellorder.cpp @@ -34,6 +34,10 @@ #include #endif +#if !CORENRN_BUILD +void verify_structure(); +#endif + #if CORENRN_BUILD namespace coreneuron { #else @@ -119,6 +123,12 @@ void destroy_interleave_info() { } } +#if !CORENRN_BUILD && defined(NRN_ENABLE_GPU) +int interleave_ncell_for_thread(int ith) { + return nrn_threads[ith].ncell; +} +#endif + // more precise visualization of the warp quality // can be called after admin2 static void print_quality2(int iwarp, InterleaveInfo& ii, int* p) { @@ -336,6 +346,9 @@ static void prnode(const char* mes, NrnThread& nt) { int nrn_optimize_node_order(int type) { if (type != interleave_permute_type) { tree_changed = 1; // calls setup_topology. v_stucture_change = 1 may be better. + interleave_permute_type = type; + ::verify_structure(); + return type; } interleave_permute_type = type; return type; @@ -528,37 +541,44 @@ void mk_cell_indices() { } #endif // INTERLEAVE_DEBUG +#define GPU_A(i) vec_a[i] +#define GPU_B(i) vec_b[i] +#define GPU_D(i) vec_d[i] +#define GPU_RHS(i) vec_rhs[i] +#define GPU_PARENT(i) nt->_v_parent_index[i] + +static void node_matrix_pointers(NrnThread* nt, + double*& vec_a, + double*& vec_b, + double*& vec_d, + double*& vec_rhs) { #if CORENRN_BUILD -#define GPU_V(i) nt->_actual_v[i] -#define GPU_A(i) nt->_actual_a[i] -#define GPU_B(i) nt->_actual_b[i] -#define GPU_D(i) nt->_actual_d[i] -#define GPU_RHS(i) nt->_actual_rhs[i] + vec_a = nt->_actual_a; + vec_b = nt->_actual_b; + vec_d = nt->_actual_d; + vec_rhs = nt->_actual_rhs; #else -#define GPU_V(i) vec_v[i] -#define GPU_A(i) vec_a[i] -#define GPU_B(i) vec_b[i] -#define GPU_D(i) vec_d[i] -#define GPU_RHS(i) vec_rhs[i] + vec_a = nt->node_a_storage(); + vec_b = nt->node_b_storage(); + vec_d = nt->node_d_storage(); + vec_rhs = nt->node_rhs_storage(); #endif -#define GPU_PARENT(i) nt->_v_parent_index[i] +} // How does the interleaved permutation with stride get used in // triagularization? // each cell in parallel regardless of inhomogeneous topology static void triang_interleaved(NrnThread* nt, + double* vec_a, + double* vec_b, + double* vec_d, + double* vec_rhs, int icell, int icellsize, int nstride, int* stride, int* lastnode) { -#if !CORENRN_BUILD - auto* const vec_a = nt->node_a_storage(); - auto* const vec_b = nt->node_b_storage(); - auto* const vec_d = nt->node_d_storage(); - auto* const vec_rhs = nt->node_rhs_storage(); -#endif int i = lastnode[icell]; for (int istride = nstride - 1; istride >= 0; --istride) { if (istride < icellsize) { // only first icellsize strides matter @@ -577,17 +597,15 @@ static void triang_interleaved(NrnThread* nt, // back substitution? static void bksub_interleaved(NrnThread* nt, + double* vec_a, + double* vec_b, + double* vec_d, + double* vec_rhs, int icell, int icellsize, int /* nstride */, int* stride, int* firstnode) { -#if !CORENRN_BUILD - auto* const vec_a = nt->node_a_storage(); - auto* const vec_b = nt->node_b_storage(); - auto* const vec_d = nt->node_d_storage(); - auto* const vec_rhs = nt->node_rhs_storage(); -#endif int i = firstnode[icell]; GPU_RHS(icell) /= GPU_D(icell); // the root for (int istride = 0; istride < icellsize; ++istride) { @@ -601,20 +619,18 @@ static void bksub_interleaved(NrnThread* nt, } } -nrn_pragma_acc(routine vector) +nrn_pragma_acc(routine seq) static void solve_interleaved2_loop_body(NrnThread* nt, + double* vec_a, + double* vec_b, + double* vec_d, + double* vec_rhs, int icore, int* ncycles, int* strides, int* stridedispl, int* rootbegin, int* nodebegin) { -#if !CORENRN_BUILD - auto* const vec_a = nt->node_a_storage(); - auto* const vec_b = nt->node_b_storage(); - auto* const vec_d = nt->node_d_storage(); - auto* const vec_rhs = nt->node_rhs_storage(); -#endif int iwarp = icore / warpsize; // figure out the >> value int ic = icore & (warpsize - 1); // figure out the & mask int ncycle = ncycles[iwarp]; @@ -709,67 +725,100 @@ void solve_interleaved2(int ith) { int ncore = nwarp * warpsize; -#ifdef _OPENACC +#if defined(_OPENACC) && CORENRN_BUILD if (corenrn_param.gpu && corenrn_param.cuda_interface) { auto* d_nt = static_cast(acc_deviceptr(nt)); auto* d_info = static_cast(acc_deviceptr(interleave_info + ith)); solve_interleaved2_launcher(d_nt, d_info, ncore, acc_get_cuda_stream(nt->stream_id)); - } else { + return; + } #endif - int* ncycles = ii.cellsize; // nwarp of these - int* stridedispl = ii.stridedispl; // nwarp+1 of these - int* strides = ii.stride; // sum ncycles of these (bad since ncompart/warpsize) - int* rootbegin = ii.firstnode; // nwarp+1 of these - int* nodebegin = ii.lastnode; // nwarp+1 of these -#if defined(CORENEURON_ENABLE_GPU) - int nstride = stridedispl[nwarp]; + // NEURON native CUDA launcher (coreneuron::solve_interleaved2_launcher) is wired when + // neuron::gpu::use_cuda_launcher() is enabled in PR 9 after device upload provides + // CoreNEURON-compatible NrnThread arena pointers. + + int* ncycles = ii.cellsize; // nwarp of these + int* stridedispl = ii.stridedispl; // nwarp+1 of these + int* strides = ii.stride; // sum ncycles of these (bad since ncompart/warpsize) + int* rootbegin = ii.firstnode; // nwarp+1 of these + int* nodebegin = ii.lastnode; // nwarp+1 of these +#if defined(CORENEURON_ENABLE_GPU) || defined(NRN_ENABLE_GPU) + int nstride = stridedispl[nwarp]; #endif - // nvc++/22.3 does not respect an if clause inside nrn_pragma_omp... -#if CORENRN_BUILD - if (nt->compute_gpu) { -#else - // bad clang-format - if (0) { // nt->compute_gpu) { + + double* vec_a{}; + double* vec_b{}; + double* vec_d{}; + double* vec_rhs{}; + node_matrix_pointers(nt, vec_a, vec_b, vec_d, vec_rhs); + +#if defined(_OPENACC) && (CORENRN_BUILD || defined(NRN_ENABLE_GPU)) + if (nt->compute_gpu) { + /* If we compare this loop with the one from cellorder.cu (CUDA version), we will + * understand that the parallelism here is exposed in steps, while in the CUDA version + * all the parallelism is exposed from the very beginning of the loop. In more details, + * here we initially distribute the outermost loop, e.g. in the CUDA blocks, and for the + * innermost loops we explicitly use multiple threads for the parallelization (see for + * example the loop directives in triang/bksub_interleaved2). On the other hand, in the + * CUDA version the outermost loop is distributed to all the available threads, and + * therefore there is no need to have the innermost loops. Here, the loop/icore jumps + * every warpsize, while in the CUDA version the icore increases by one. Other than + * this, the two loop versions are equivalent (same results). + */ + nrn_pragma_acc(parallel loop gang present(nt [0:1], + vec_a [0:nt->end], + vec_b [0:nt->end], + vec_d [0:nt->end], + vec_rhs [0:nt->end], + strides [0:nstride], + ncycles [0:nwarp], + stridedispl [0:nwarp + 1], + rootbegin [0:nwarp + 1], + nodebegin [0:nwarp + 1]) async(nt->stream_id)) + // clang-format off + nrn_pragma_omp(target teams loop map(present, alloc: nt[:1], + vec_a[:nt->end], + vec_b[:nt->end], + vec_d[:nt->end], + vec_rhs[:nt->end], + strides[:nstride], + ncycles[:nwarp], + stridedispl[:nwarp + 1], + rootbegin[:nwarp + 1], + nodebegin[:nwarp + 1])) + // clang-format on + for (int icore = 0; icore < ncore; icore += warpsize) { + solve_interleaved2_loop_body(nt, + vec_a, + vec_b, + vec_d, + vec_rhs, + icore, + ncycles, + strides, + stridedispl, + rootbegin, + nodebegin); + } + nrn_pragma_acc(wait(nt->stream_id)) + } else #endif - /* If we compare this loop with the one from cellorder.cu (CUDA version), we will - * understand that the parallelism here is exposed in steps, while in the CUDA version - * all the parallelism is exposed from the very beginning of the loop. In more details, - * here we initially distribute the outermost loop, e.g. in the CUDA blocks, and for the - * innermost loops we explicitly use multiple threads for the parallelization (see for - * example the loop directives in triang/bksub_interleaved2). On the other hand, in the - * CUDA version the outermost loop is distributed to all the available threads, and - * therefore there is no need to have the innermost loops. Here, the loop/icore jumps - * every warpsize, while in the CUDA version the icore increases by one. Other than - * this, the two loop versions are equivalent (same results). - */ - nrn_pragma_acc(parallel loop gang present(nt [0:1], - strides [0:nstride], - ncycles [0:nwarp], - stridedispl [0:nwarp + 1], - rootbegin [0:nwarp + 1], - nodebegin [0:nwarp + 1]) async(nt->stream_id)) - // clang-format off - nrn_pragma_omp(target teams loop map(present, alloc: nt[:1], - strides[:nstride], - ncycles[:nwarp], - stridedispl[:nwarp + 1], - rootbegin[:nwarp + 1], - nodebegin[:nwarp + 1])) - // clang-format on - for (int icore = 0; icore < ncore; icore += warpsize) { - solve_interleaved2_loop_body( - nt, icore, ncycles, strides, stridedispl, rootbegin, nodebegin); - } - nrn_pragma_acc(wait(nt->stream_id)) - } else { - for (int icore = 0; icore < ncore; icore += warpsize) { - solve_interleaved2_loop_body( - nt, icore, ncycles, strides, stridedispl, rootbegin, nodebegin); - } + { + // NEURON native host fallback when compute_gpu is off or OpenACC is unavailable. + for (int icore = 0; icore < ncore; icore += warpsize) { + solve_interleaved2_loop_body(nt, + vec_a, + vec_b, + vec_d, + vec_rhs, + icore, + ncycles, + strides, + stridedispl, + rootbegin, + nodebegin); } -#ifdef _OPENACC } -#endif } /** @@ -792,10 +841,21 @@ void solve_interleaved1(int ith) { int* lastnode = ii.lastnode; int* cellsize = ii.cellsize; + double* vec_a{}; + double* vec_b{}; + double* vec_d{}; + double* vec_rhs{}; + node_matrix_pointers(nt, vec_a, vec_b, vec_d, vec_rhs); + +#if defined(_OPENACC) && (CORENRN_BUILD || defined(NRN_ENABLE_GPU)) // OL211123: can we preserve the error checking behaviour of OpenACC's // present clause with OpenMP? It is a bug if these data are not present, // so diagnostics are helpful... nrn_pragma_acc(parallel loop present(nt [0:1], + vec_a [0:nt->end], + vec_b [0:nt->end], + vec_d [0:nt->end], + vec_rhs [0:nt->end], stride [0:nstride], firstnode [0:ncell], lastnode [0:ncell], @@ -804,10 +864,21 @@ void solve_interleaved1(int ith) { nrn_pragma_omp(target teams distribute parallel for simd if(nt->compute_gpu)) for (int icell = 0; icell < ncell; ++icell) { int icellsize = cellsize[icell]; - triang_interleaved(nt, icell, icellsize, nstride, stride, lastnode); - bksub_interleaved(nt, icell, icellsize, nstride, stride, firstnode); + triang_interleaved( + nt, vec_a, vec_b, vec_d, vec_rhs, icell, icellsize, nstride, stride, lastnode); + bksub_interleaved( + nt, vec_a, vec_b, vec_d, vec_rhs, icell, icellsize, nstride, stride, firstnode); } nrn_pragma_acc(wait(nt->stream_id)) +#else + for (int icell = 0; icell < ncell; ++icell) { + int icellsize = cellsize[icell]; + triang_interleaved( + nt, vec_a, vec_b, vec_d, vec_rhs, icell, icellsize, nstride, stride, lastnode); + bksub_interleaved( + nt, vec_a, vec_b, vec_d, vec_rhs, icell, icellsize, nstride, stride, firstnode); + } +#endif } void solve_interleaved(int ith) { diff --git a/src/coreneuron/permute/cellorder.hpp b/src/coreneuron/permute/cellorder.hpp index 353d9dd077..e8fd6acae9 100644 --- a/src/coreneuron/permute/cellorder.hpp +++ b/src/coreneuron/permute/cellorder.hpp @@ -38,6 +38,11 @@ std::vector interleave_order(int ith, int ncell, int nnode, int* parent); void create_interleave_info(); void destroy_interleave_info(); +#if !CORENRN_BUILD && defined(NRN_ENABLE_GPU) +/** Number of cells on NrnThread @p ith (for InterleaveInfo permute-1 upload). */ +int interleave_ncell_for_thread(int ith); +#endif + #if CORENRN_BUILD /** * @@ -56,7 +61,7 @@ class InterleaveInfo; // forward declaration * * \brief CUDA branch of the solve_interleaved with interleave_permute_type == 2. * - * This branch is activated in runtime with the --cuda-interface CLI flag + * Activated in runtime with the --cuda-interface CLI flag. */ void solve_interleaved2_launcher(NrnThread* nt, InterleaveInfo* info, int ncore, void* stream); #endif @@ -145,4 +150,4 @@ void copy_align_array(T*& dest, T* src, size_t n) { #if INTERLEAVE_DEBUG void mk_cell_indices(); #endif -} // namespace coreneuron +} // namespace coreneuron/neuron diff --git a/src/ivoc/graph.cpp b/src/ivoc/graph.cpp index dfa62e1007..c730f6b0aa 100644 --- a/src/ivoc/graph.cpp +++ b/src/ivoc/graph.cpp @@ -33,6 +33,9 @@ extern Image* gif_image(const char*); #include "graph.h" #include "axis.h" #include "hocmark.h" +#if defined(NRN_ENABLE_GPU) +#include "neuron/gpu/download.hpp" +#endif #include "mymath.h" #include "idraw.h" #include "symchoos.h" @@ -1988,6 +1991,9 @@ void Graph::line(Coord x, Coord y) { current_polyline_->plot(x, y); } void Graph::flush() { +#if defined(NRN_ENABLE_GPU) + neuron::gpu::batch_download_to_host(); +#endif extension_start(); long i, cnt = count(); for (i = 0; i < cnt; ++i) { diff --git a/src/neuron/CMakeLists.txt b/src/neuron/CMakeLists.txt new file mode 100644 index 0000000000..4c7d3a57f6 --- /dev/null +++ b/src/neuron/CMakeLists.txt @@ -0,0 +1,57 @@ +# ============================================================================= +# NEURON 9.0 model-data layer (src/neuron/) — GPU offload scaffold and future native GPU engine +# sources. +# ============================================================================= +add_library(neuron_gpu STATIC gpu/offload.cpp gpu/device_state.cpp gpu/config.cpp + gpu/device_assign.cpp) +set_property(TARGET neuron_gpu PROPERTY POSITION_INDEPENDENT_CODE ON) +target_include_directories(neuron_gpu PUBLIC ${PROJECT_SOURCE_DIR}/src) +target_link_libraries(neuron_gpu PRIVATE fmt::fmt) + +cpp_cc_configure_sanitizers(TARGET neuron_gpu) +if(NRN_ENABLE_GPU) + target_compile_definitions(neuron_gpu PUBLIC NRN_ENABLE_GPU) +endif() +if(NRN_ENABLE_GPU AND CORENRN_ENABLE_GPU) + separate_arguments(_neuron_gpu_acc_flags UNIX_COMMAND "${CORENRN_ACC_COMP_FLAGS}") + set(_neuron_gpu_openacc_sources gpu/offload.cpp) + set_source_files_properties( + ${_neuron_gpu_openacc_sources} + PROPERTIES COMPILE_OPTIONS "${_neuron_gpu_acc_flags}" COMPILE_DEFINITIONS + "CORENEURON_ENABLE_GPU;NRN_ENABLE_GPU") + + # upload.cpp pulls NEURON/oc headers; compile as OpenACC object lib for libnrniv + GPU tests. + add_library( + neuron_gpu_upload OBJECT + gpu/upload.cpp + gpu/upload_mechanisms.cpp + gpu/download.cpp + gpu/fadvance_gpu.cpp + gpu/net_events.cpp + gpu/net_send_buffer.cpp + gpu/post_solve.cpp + gpu/sync.cpp) + set_property(TARGET neuron_gpu_upload PROPERTY POSITION_INDEPENDENT_CODE ON) + target_include_directories( + neuron_gpu_upload + PRIVATE ${PROJECT_SOURCE_DIR}/src + ${PROJECT_SOURCE_DIR}/src/nrniv + ${PROJECT_SOURCE_DIR}/src/nrnoc + ${PROJECT_SOURCE_DIR}/src/oc + ${PROJECT_BINARY_DIR} + ${PROJECT_BINARY_DIR}/src/nrnoc + ${PROJECT_BINARY_DIR}/src/oc + ${PROJECT_SOURCE_DIR}/external/fmt/include) + target_link_libraries(neuron_gpu_upload PRIVATE fmt::fmt) + set_source_files_properties( + gpu/upload.cpp + gpu/upload_mechanisms.cpp + gpu/download.cpp + gpu/fadvance_gpu.cpp + gpu/net_events.cpp + gpu/net_send_buffer.cpp + gpu/post_solve.cpp + gpu/sync.cpp + PROPERTIES COMPILE_OPTIONS "${_neuron_gpu_acc_flags}" COMPILE_DEFINITIONS + "CORENEURON_ENABLE_GPU;NRN_ENABLE_GPU") +endif() diff --git a/src/neuron/cache/model_data.hpp b/src/neuron/cache/model_data.hpp index 675b9b4097..826ad47325 100644 --- a/src/neuron/cache/model_data.hpp +++ b/src/neuron/cache/model_data.hpp @@ -26,6 +26,9 @@ struct Mechanism { struct Thread { /** * @brief Offset into global Node storage for this thread. + * + * Used by native GPU upload (neuron::gpu::upload_sorted_model) to present + * per-thread views of contiguous SOA node vectors. */ std::size_t node_data_offset{}; /** diff --git a/src/neuron/container/soa_container.hpp b/src/neuron/container/soa_container.hpp index c24ccb42dc..8d94a741e3 100644 --- a/src/neuron/container/soa_container.hpp +++ b/src/neuron/container/soa_container.hpp @@ -651,6 +651,11 @@ struct state_token { m_container->decrease_frozen_count(); } + [[nodiscard]] Container& container() const { + assert(m_container); + return *m_container; + } + private: template friend struct soa; @@ -1546,6 +1551,14 @@ struct soa { return accumulated.usage(); } +#if defined(NRN_ENABLE_GPU) + /** @brief Const iteration over SOA vectors for native GPU upload (PR 9). */ + template + Callable for_each_vector_for_gpu_upload(Callable callable) const { + return for_each_vector(callable); + } +#endif + private: /** * @brief Throw an exception with a pretty prefix. diff --git a/src/neuron/gpu/config.cpp b/src/neuron/gpu/config.cpp new file mode 100644 index 0000000000..6e39460542 --- /dev/null +++ b/src/neuron/gpu/config.cpp @@ -0,0 +1,175 @@ +#include "neuron/gpu/config.hpp" + +#include "node_order_optim/node_order_optim.h" + +#include +#include +#include +#include +#include + +extern int cvode_active_; +extern int nrn_nthread; + +namespace neuron::gpu { +namespace { + +struct RuntimeConfig { + bool enable{false}; + Backend backend{Backend::Coreneuron}; + unsigned device_count{0}; +}; + +RuntimeConfig& config() { + static RuntimeConfig instance; + return instance; +} + +Backend parse_backend(std::string_view name) { + std::string lowered{name}; + std::transform(lowered.begin(), lowered.end(), lowered.begin(), [](unsigned char c) { + return static_cast(std::tolower(c)); + }); + if (lowered == "native") { + return Backend::Native; + } + if (lowered == "coreneuron") { + return Backend::Coreneuron; + } + throw std::invalid_argument("neuron::gpu::set_backend: expected 'native' or 'coreneuron'"); +} + +} // namespace + +bool enabled() noexcept { +#if defined(NRN_ENABLE_GPU) + return config().enable; +#else + return false; +#endif +} + +bool backend_native() noexcept { +#if defined(NRN_ENABLE_GPU) + return config().backend == Backend::Native; +#else + return false; +#endif +} + +Backend backend() noexcept { + return config().backend; +} + +void ensure_native_gpu_cell_permute() noexcept { +#if defined(NRN_ENABLE_GPU) + if (!enabled() || !backend_native()) { + return; + } + if (neuron::interleave_permute_type == 2) { + return; + } + int const previous = neuron::interleave_permute_type; + neuron::nrn_optimize_node_order(2); + if (previous != 2) { + fprintf(stderr, + "neuron::gpu: native GPU requires cell permute type 2 " + "(interleave_permute_type was %d); using permute 2\n", + previous); + } +#else +#endif +} + +void set_enable(bool value) noexcept { +#if defined(NRN_ENABLE_GPU) + config().enable = value; + if (value) { + ensure_native_gpu_cell_permute(); + } +#else + (void) value; +#endif +} + +void set_backend(std::string_view name) { +#if defined(NRN_ENABLE_GPU) + config().backend = parse_backend(name); + ensure_native_gpu_cell_permute(); +#else + (void) name; +#endif +} + +unsigned device_count() noexcept { +#if defined(NRN_ENABLE_GPU) + return config().device_count; +#else + return 0; +#endif +} + +void set_device_count(unsigned value) noexcept { +#if defined(NRN_ENABLE_GPU) + config().device_count = value; +#else + (void) value; +#endif +} + +bool use_cuda_launcher() noexcept { +#if defined(NRN_ENABLE_GPU) + return enabled() && backend_native(); +#else + return false; +#endif +} + +void warn_native_gpu_multithread_policy() noexcept { +#if defined(NRN_ENABLE_GPU) + static bool warned{false}; + if (warned || !enabled() || !backend_native() || nrn_nthread <= 1) { + return; + } + warned = true; + fprintf(stderr, + "neuron::gpu: native GPU is validated primarily with pc.nthread(1); " + "pc.nthread(%d) may fail with per-thread OpenACC CUDA contexts\n", + nrn_nthread); +#endif +} + +const char* native_gpu_configuration_error() noexcept { +#if defined(NRN_ENABLE_GPU) + if (!enabled() || !backend_native()) { + return nullptr; + } + if (cvode_active_) { + return "native GPU backend (gpu.backend='native') requires fixed-step integration; " + "deactivate CVode before enabling native GPU"; + } +#endif + return nullptr; +} + +namespace detail { + +void reset_config_for_testing() { + config() = RuntimeConfig{}; +} + +void set_enable_for_testing(bool value) { + config().enable = value; +} + +void set_backend_for_testing(Backend value) { + config().backend = value; +} + +void set_device_count_for_testing(unsigned value) { + config().device_count = value; +} + +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/config.hpp b/src/neuron/gpu/config.hpp new file mode 100644 index 0000000000..bc8caf87cf --- /dev/null +++ b/src/neuron/gpu/config.hpp @@ -0,0 +1,62 @@ +#pragma once + +#include + +namespace neuron::gpu { + +enum class Backend { + Coreneuron, + Native, +}; + +/** True when gpu.enable is set and the build has NRN_ENABLE_GPU. */ +bool enabled() noexcept; + +/** True when gpu.backend is native. */ +bool backend_native() noexcept; + +/** Current runtime backend selection. */ +Backend backend() noexcept; + +/** @brief Set gpu.enable (no-op when NRN_ENABLE_GPU is off). */ +void set_enable(bool value) noexcept; + +/** @brief Set gpu.backend from "native" or "coreneuron". */ +void set_backend(std::string_view name); + +/** Number of GPUs to use per node (0 = all available). */ +unsigned device_count() noexcept; + +/** @brief Set gpu.device_count (0 = all available). */ +void set_device_count(unsigned value) noexcept; + +/** + * True when solve_interleaved2 should use the CUDA launcher (cellorder.cu). + * Requires native backend with device upload; scaffolding returns false until PR 12. + */ +bool use_cuda_launcher() noexcept; + +/** + * Phase B contract check for native GPU. Returns nullptr when configuration is + * valid, otherwise a stable error message (fixed-step only; CVode must be off). + */ +[[nodiscard]] const char* native_gpu_configuration_error() noexcept; + +/** One-time stderr notice when native GPU runs with pc.nthread() > 1. */ +void warn_native_gpu_multithread_policy() noexcept; + +/** + * Native GPU fixed-step requires interleaved Hines solve (permute type 2). + * Called when gpu.enable and gpu.backend="native" are both active; sets + * interleave_permute_type to 2 if it is not already. + */ +void ensure_native_gpu_cell_permute() noexcept; + +namespace detail { +void reset_config_for_testing(); +void set_enable_for_testing(bool value); +void set_backend_for_testing(Backend value); +void set_device_count_for_testing(unsigned value); +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/device_assign.cpp b/src/neuron/gpu/device_assign.cpp new file mode 100644 index 0000000000..4245991ea7 --- /dev/null +++ b/src/neuron/gpu/device_assign.cpp @@ -0,0 +1,117 @@ +#include "neuron/gpu/device_assign.hpp" + +#include "neuron/gpu/config.hpp" +#include "neuron/gpu/offload.hpp" + +#include +#include +#include + +#if NRNMPI +#include +extern int nrnmpi_use __attribute__((weak)); +extern int nrnmpi_myid __attribute__((weak)); +#endif + +namespace neuron::gpu { +namespace { + +int mpi_simulation_rank() { +#if NRNMPI + if (&nrnmpi_myid != nullptr) { + return nrnmpi_myid; + } +#endif + return 0; +} + +int mpi_local_rank() { +#if NRNMPI + if (&nrnmpi_use != nullptr && nrnmpi_use != 0) { + MPI_Comm local_comm{}; + MPI_Comm_split_type(MPI_COMM_WORLD, + MPI_COMM_TYPE_SHARED, + mpi_simulation_rank(), + MPI_INFO_NULL, + &local_comm); + int local_rank = 0; + MPI_Comm_rank(local_comm, &local_rank); + MPI_Comm_free(&local_comm); + return local_rank; + } +#endif + return 0; +} + +int mpi_local_size() { +#if NRNMPI + if (&nrnmpi_use != nullptr && nrnmpi_use != 0) { + MPI_Comm local_comm{}; + MPI_Comm_split_type(MPI_COMM_WORLD, + MPI_COMM_TYPE_SHARED, + mpi_simulation_rank(), + MPI_INFO_NULL, + &local_comm); + int local_size = 1; + MPI_Comm_size(local_comm, &local_size); + MPI_Comm_free(&local_comm); + return local_size; + } +#endif + return 1; +} + +std::atomic g_assigned_device_id{-1}; +std::atomic g_device_assigned{false}; + +} // namespace + +void assign_device() { +#if defined(NRN_ENABLE_GPU) + if (g_device_assigned.exchange(true)) { + return; + } + + int num_devices_per_node = target_get_num_devices(); + if (num_devices_per_node == 0) { + throw std::runtime_error( + "neuron::gpu::assign_device: GPU execution enabled but no NVIDIA GPU found"); + } + + auto const requested = device_count(); + if (requested != 0) { + if (static_cast(requested) > num_devices_per_node) { + throw std::runtime_error( + "neuron::gpu::assign_device: requested device_count exceeds available GPUs"); + } + num_devices_per_node = static_cast(requested); + } + + int const local_rank = mpi_local_rank(); + int const device_id = local_rank % num_devices_per_node; + target_set_default_device(device_id); + g_assigned_device_id.store(device_id); + + if (mpi_simulation_rank() == 0) { + std::cout << " Info : " << num_devices_per_node << " GPUs shared by " << mpi_local_size() + << " ranks per node\n"; + } +#else + (void) 0; +#endif +} + +int assigned_device_id() noexcept { + return g_assigned_device_id.load(); +} + +namespace detail { + +void reset_device_assignment_for_testing() { + g_device_assigned.store(false); + g_assigned_device_id.store(-1); +} + +} // namespace detail + +} // namespace neuron::gpu \ No newline at end of file diff --git a/src/neuron/gpu/device_assign.hpp b/src/neuron/gpu/device_assign.hpp new file mode 100644 index 0000000000..eabadb1ee4 --- /dev/null +++ b/src/neuron/gpu/device_assign.hpp @@ -0,0 +1,15 @@ +#pragma once + +namespace neuron::gpu { + +/** Assign the default OpenACC/CUDA device (idempotent). */ +void assign_device(); + +/** Device id selected by the last assign_device() call, or -1 if not yet assigned. */ +int assigned_device_id() noexcept; + +namespace detail { +void reset_device_assignment_for_testing(); +} // namespace detail + +} // namespace neuron::gpu \ No newline at end of file diff --git a/src/neuron/gpu/device_state.cpp b/src/neuron/gpu/device_state.cpp new file mode 100644 index 0000000000..b128b59a94 --- /dev/null +++ b/src/neuron/gpu/device_state.cpp @@ -0,0 +1,248 @@ +#include "neuron/gpu/device_state.hpp" + +#include "neuron/gpu/config.hpp" +#include "neuron/gpu/device_assign.hpp" +#include "neuron/gpu/download.hpp" +#include "neuron/gpu/upload.hpp" + +#include +#include +#include + +namespace neuron::gpu { + +namespace { + +struct ModelDeviceState { + std::atomic sorted_token_refs{0}; + std::atomic device_token_refs{0}; + bool on_device{false}; + UploadState upload{}; + + void upload_model(model_sorted_token const& sorted) { + upload_sorted_model(sorted, upload); + on_device = true; + } + + void download_to_host() { + batch_download_to_host(); + } + + void upload_to_device() { + batch_upload_to_device(); + } + + void teardown() { + upload.teardown(); + on_device = false; + } +}; + +class DeviceStateRegistry { + public: + std::shared_ptr state() { + std::lock_guard lock{mut_}; + if (!active_) { + active_ = std::make_shared(); + } + return active_; + } + + void invalidate() { + std::lock_guard lock{mut_}; + if (active_) { + active_->teardown(); + } + cached_ensure_token_ = nullptr; + cached_ensure_storage_.reset(); + } + + void on_sorted_token_created() { + auto dev_state = this->state(); + ++dev_state->sorted_token_refs; + } + + void on_sorted_token_destroyed() { + std::shared_ptr state; + { + std::lock_guard lock{mut_}; + state = active_; + if (!state) { + return; + } + auto const remaining = --state->sorted_token_refs; + if (remaining > 0) { + return; + } + cached_ensure_token_ = nullptr; + cached_ensure_storage_.reset(); + if (state->device_token_refs > 0) { + throw std::runtime_error( + "neuron::gpu: model_sorted_token destroyed while device_token(s) still alive"); + } + state->teardown(); + reset_active_locked(); + } + } + + void on_device_token_created(std::shared_ptr const& state) { + ++state->device_token_refs; + } + + void on_device_token_destroyed(std::shared_ptr const& state) { + if (--state->device_token_refs > 0) { + return; + } + std::lock_guard lock{mut_}; + if (active_.get() != state.get()) { + return; + } + if (state->sorted_token_refs > 0) { + return; + } + state->teardown(); + reset_active_locked(); + } + + device_token const& ensure_cached(model_sorted_token const& sorted) { + // Hold the lock for the full upload: concurrent device_token construction + // would otherwise race on the shared UploadState mirror list. + std::lock_guard lock{mut_}; + if (!cached_ensure_token_) { + cached_ensure_storage_ = std::make_unique(sorted); + cached_ensure_token_ = cached_ensure_storage_.get(); + } + return *cached_ensure_token_; + } + + std::size_t sorted_token_count() const { + std::lock_guard lock{mut_}; + return active_ ? active_->sorted_token_refs.load() : 0; + } + + bool is_on_device() const { + std::lock_guard lock{mut_}; + return active_ && active_->on_device; + } + + std::size_t upload_mirror_count() const { + std::lock_guard lock{mut_}; + return active_ ? active_->upload.mirror_count() : 0; + } + + bool upload_is_present(void const* host_ptr) const { + std::lock_guard lock{mut_}; + return active_ && active_->upload.is_present(host_ptr); + } + + private: + void reset_active_locked() { + active_.reset(); + cached_ensure_token_ = nullptr; + cached_ensure_storage_.reset(); + } + + mutable std::recursive_mutex mut_{}; + std::shared_ptr active_{}; + device_token const* cached_ensure_token_{nullptr}; + std::unique_ptr cached_ensure_storage_{}; +}; + +DeviceStateRegistry& registry() { + static DeviceStateRegistry instance; + return instance; +} + +} // namespace + +struct device_token::State { + std::shared_ptr model_state; +}; + +device_token::device_token(model_sorted_token const& sorted) + : m_state{std::make_shared()} { + auto model_state = registry().state(); + if (model_state->sorted_token_refs == 0) { + throw std::runtime_error("neuron::gpu::device_token requires an active model_sorted_token"); + } + m_state->model_state = std::move(model_state); + registry().on_device_token_created(m_state->model_state); + if (!m_state->model_state->on_device) { +#if defined(NRN_ENABLE_GPU) + if (enabled()) { + assign_device(); + } +#endif + m_state->model_state->upload_model(sorted); + } +} + +device_token::device_token(device_token const& other) + : m_state{other.m_state} { + if (m_state) { + registry().on_device_token_created(m_state->model_state); + } +} + +device_token::device_token(device_token&& other) noexcept + : m_state{std::move(other.m_state)} {} + +device_token::~device_token() { + if (m_state) { + registry().on_device_token_destroyed(m_state->model_state); + } +} + +bool device_token::is_on_device() const { + return m_state && m_state->model_state && m_state->model_state->on_device; +} + +void device_token::update_host() { + if (m_state && m_state->model_state) { + m_state->model_state->download_to_host(); + } +} + +void device_token::update_device() { + if (m_state && m_state->model_state) { + m_state->model_state->upload_to_device(); + } +} + +device_token const& ensure_on_device(model_sorted_token const& sorted) { + return registry().ensure_cached(sorted); +} + +void invalidate_device_state() { + registry().invalidate(); +} + +namespace detail { + +void on_sorted_token_created() { + registry().on_sorted_token_created(); +} + +void on_sorted_token_destroyed() { + registry().on_sorted_token_destroyed(); +} + +std::size_t sorted_token_count_for_testing() { + return registry().sorted_token_count(); +} + +bool is_on_device_for_testing() { + return registry().is_on_device(); +} + +std::size_t mirror_count_for_testing() { + return registry().upload_mirror_count(); +} + +bool is_present_for_testing(void const* host_ptr) { + return registry().upload_is_present(host_ptr); +} + +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/device_state.hpp b/src/neuron/gpu/device_state.hpp new file mode 100644 index 0000000000..944fd36e00 --- /dev/null +++ b/src/neuron/gpu/device_state.hpp @@ -0,0 +1,69 @@ +#pragma once + +#include +#include + +namespace neuron { +struct model_sorted_token; +} + +namespace neuron::gpu { + +/** + * @brief RAII handle for GPU mirrors of a sorted model layout. + * + * Lifetime is tied to model_sorted_token / frozen-token refcounting: GPU teardown + * runs when the last model_sorted_token for the active layout is destroyed. + * Multiple device_token instances may share the same upload (refcounted). + */ +class device_token { + public: + explicit device_token(model_sorted_token const& sorted); + device_token(device_token const& other); + device_token(device_token&& other) noexcept; + device_token& operator=(device_token const&) = delete; + device_token& operator=(device_token&&) = delete; + ~device_token(); + + [[nodiscard]] bool is_on_device() const; + + /** @brief Pull recorded GPU state (voltages, fast_imem) to the host. */ + void update_host(); + + /** @brief Push host voltages to the device after HOC/VecPlay writes. */ + void update_device(); + + private: + struct State; + std::shared_ptr m_state; +}; + +/** + * @brief Upload sorted SOA vectors to the device on first GPU step. + * + * Returns a device_token referencing the shared upload state for the current + * sorted layout. Safe to call repeatedly; upload happens at most once per layout. + */ +[[nodiscard]] device_token const& ensure_on_device(model_sorted_token const& sorted); + +/** @brief Discard GPU mirrors when the sorted layout is invalidated. */ +void invalidate_device_state(); + +namespace detail { +void on_sorted_token_created(); +void on_sorted_token_destroyed(); + +/** @brief Test hook: number of live model_sorted_token registrations. */ +[[nodiscard]] std::size_t sorted_token_count_for_testing(); + +/** @brief Test hook: whether the active layout has been uploaded. */ +[[nodiscard]] bool is_on_device_for_testing(); + +/** @brief Test hook: number of device mirrors recorded for the active layout. */ +[[nodiscard]] std::size_t mirror_count_for_testing(); + +/** @brief Test hook: whether a host pointer was copyin'd for the active layout. */ +[[nodiscard]] bool is_present_for_testing(void const* host_ptr); +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/download.cpp b/src/neuron/gpu/download.cpp new file mode 100644 index 0000000000..c5d62cee69 --- /dev/null +++ b/src/neuron/gpu/download.cpp @@ -0,0 +1,75 @@ +#include "neuron/gpu/download.hpp" + +#include "multicore.h" +#include "neuron/gpu/config.hpp" +#include "neuron/gpu/sync.hpp" + +namespace neuron::gpu { +namespace { + +std::size_t g_flush_interval{1}; +std::size_t g_step_counter{0}; + +} // namespace + +std::size_t download_flush_interval() noexcept { + return g_flush_interval; +} + +void set_download_flush_interval(std::size_t interval) noexcept { + g_flush_interval = interval; +} + +void reset_download_step_counter() noexcept { + g_step_counter = 0; +} + +void advance_download_step_counter() noexcept { + ++g_step_counter; +} + +bool should_flush_download() noexcept { + if (g_flush_interval == 0) { + return false; + } + return (g_step_counter % g_flush_interval) == 0; +} + +void batch_download_post_solve(NrnThread& nt) { + sync_voltages_to_host_after_post_solve(nt); + sync_fast_imem_to_host_after_post_solve(nt); +} + +void batch_download_to_host() { +#if defined(NRN_ENABLE_GPU) + if (!enabled() || !backend_native()) { + return; + } + for (int ith = 0; ith < nrn_nthread; ++ith) { + batch_download_post_solve(nrn_threads[ith]); + } +#endif +} + +void batch_upload_to_device() { +#if defined(NRN_ENABLE_GPU) + if (!enabled() || !backend_native()) { + return; + } + for (int ith = 0; ith < nrn_nthread; ++ith) { + sync_after_vecplay(nrn_threads[ith]); + } +#endif +} + +void finalize_psolve_download() { +#if defined(NRN_ENABLE_GPU) + if (!enabled() || !backend_native()) { + return; + } + batch_download_to_host(); + reset_download_step_counter(); +#endif +} + +} // namespace neuron::gpu \ No newline at end of file diff --git a/src/neuron/gpu/download.hpp b/src/neuron/gpu/download.hpp new file mode 100644 index 0000000000..429ecb400b --- /dev/null +++ b/src/neuron/gpu/download.hpp @@ -0,0 +1,35 @@ +#pragma once + +#include + +struct NrnThread; + +namespace neuron::gpu { + +/** Steps between host downloads during psolve (0 = only at psolve end). */ +[[nodiscard]] std::size_t download_flush_interval() noexcept; + +void set_download_flush_interval(std::size_t interval) noexcept; + +/** Reset the per-psolve step counter (call at psolve start). */ +void reset_download_step_counter() noexcept; + +/** Advance the per-psolve step counter after a fixed step. */ +void advance_download_step_counter() noexcept; + +/** True when the current step should pull recorded state to the host. */ +[[nodiscard]] bool should_flush_download() noexcept; + +/** Pull post-solve node voltages and fast_imem to the host for one thread. */ +void batch_download_post_solve(NrnThread& nt); + +/** Pull all thread state needed for HOC reads and Vector.record. */ +void batch_download_to_host(); + +/** Push host voltages to the device after HOC/VecPlay writes. */ +void batch_upload_to_device(); + +/** Final download at psolve end (always runs when native GPU is active). */ +void finalize_psolve_download(); + +} // namespace neuron::gpu \ No newline at end of file diff --git a/src/neuron/gpu/fadvance_gpu.cpp b/src/neuron/gpu/fadvance_gpu.cpp new file mode 100644 index 0000000000..2711b15953 --- /dev/null +++ b/src/neuron/gpu/fadvance_gpu.cpp @@ -0,0 +1,130 @@ +#include "neuron/gpu/fadvance_gpu.hpp" + +#include "neuron/gpu/config.hpp" +#include "neuron/gpu/device_state.hpp" +#include "neuron/gpu/net_events.hpp" +#include "neuron/gpu/net_send_buffer.hpp" +#include "neuron/gpu/download.hpp" +#include "neuron/gpu/post_solve.hpp" +#include "neuron/gpu/sync.hpp" +#include "neuron/model_data.hpp" + +#include "coreneuron/permute/cellorder.hpp" +#include "multicore.h" +#include "neuron.h" +#include "node_order_optim/node_order_optim.h" +#include "nrn_ansi.h" +#include "nrncvode.h" +#include "utils/profile/profiler_interface.h" + +#include +#include + +extern void (*nrnthread_v_transfer_)(NrnThread* nt); +extern void (*nrnmpi_v_transfer_)(); + +namespace neuron::gpu { +namespace { + +std::atomic g_fixed_step_dispatch_count{0}; + +void advance_first_half_time(NrnThread& nt) { + nt._t += .5 * nt._dt; +} + +} // namespace + +void fixed_step_thread(model_sorted_token const& cache_token, + device_token const& /*dev*/, + NrnThread& nt) { + ++g_fixed_step_dispatch_count; + if (nt.id == 0) { + warn_native_gpu_multithread_policy(); + } + auto* const nth = &nt; + + int const saved_compute_gpu = nt.compute_gpu; + nt.compute_gpu = 1; + + { + nrn::Instrumentor::phase p("deliver-events"); + deliver_net_events_host(nth); + } + + ensure_thread_net_send_buffers(nth); + nrn_random_play(); + advance_first_half_time(nt); + fixed_play_continuous(nth); + sync_after_vecplay(nt); + bool const host_post_solve = nt.end > 0 && post_solve_needs_host_fallback(nt); + if (nt.end > 0) { + setup_tree_matrix(cache_token, nt); + sync_matrix_to_device_before_solve(nt); + flush_mechanism_net_send_buffers(nth); + { + nrn::Instrumentor::phase p("matrix-solver"); + if (neuron::interleave_permute_type) { + neuron::solve_interleaved(nt.id); + } else { + nrn_solve(nth); + } + } + if (host_post_solve) { + sync_rhs_to_host_after_solve(nt); + { + nrn::Instrumentor::phase p("second-order-cur"); + second_order_cur(nth); + } + { + nrn::Instrumentor::phase p("update"); + nrn_update_voltage(cache_token, nt); + } + } else { + { + nrn::Instrumentor::phase p("update"); + post_solve_on_device(cache_token, nt); + } + if (should_flush_download()) { + batch_download_post_solve(nt); + } + } + advance_download_step_counter(); + } + if (nrnthread_v_transfer_) { + if (nt.end > 0) { + if (host_post_solve) { + // Host nrn_update_voltage already updated vec_v; push to device and + // leave host voltages intact for partrans gather (device→host would + // overwrite with stale GPU state). + sync_gap_after_host_voltage_update(nt); + } else { + sync_gap_after_voltage_update(nt); + } + if (nrnmpi_v_transfer_) { + (*nrnmpi_v_transfer_)(); + } + } + nrn_fixed_step_lastpart(cache_token, nt); + } else { + nrn_fixed_step_lastpart(cache_token, nt); + } + if (nt.end > 0) { + sync_after_vecplay(nt); + } + + nt.compute_gpu = saved_compute_gpu; +} + +namespace detail { + +std::size_t fixed_step_dispatch_count_for_testing() { + return g_fixed_step_dispatch_count.load(); +} + +void reset_fixed_step_dispatch_for_testing() { + g_fixed_step_dispatch_count.store(0); +} + +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/fadvance_gpu.hpp b/src/neuron/gpu/fadvance_gpu.hpp new file mode 100644 index 0000000000..6be1deb5ae --- /dev/null +++ b/src/neuron/gpu/fadvance_gpu.hpp @@ -0,0 +1,23 @@ +#pragma once + +namespace neuron { +struct model_sorted_token; +} + +struct NrnThread; + +namespace neuron::gpu { + +class device_token; + +/** Native GPU per-thread fixed-step body (fuses lastpart when no gap transfer). */ +void fixed_step_thread(model_sorted_token const& cache_token, + device_token const& dev, + NrnThread& nt); + +namespace detail { +[[nodiscard]] std::size_t fixed_step_dispatch_count_for_testing(); +void reset_fixed_step_dispatch_for_testing(); +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/net_events.cpp b/src/neuron/gpu/net_events.cpp new file mode 100644 index 0000000000..fd4407f187 --- /dev/null +++ b/src/neuron/gpu/net_events.cpp @@ -0,0 +1,62 @@ +#include "neuron/gpu/net_events.hpp" + +#include "neuron/gpu/config.hpp" + +#include "nrncvode.h" + +#include + +namespace neuron::gpu { +namespace { + +std::atomic g_deliver_net_events_count{0}; +std::atomic g_deliver_post_step_events_count{0}; +std::atomic g_spike_exchange_count{0}; + +} // namespace + +void deliver_net_events_host(NrnThread* nt) { + ++g_deliver_net_events_count; + deliver_net_events(nt); +} + +void deliver_post_step_events_host(NrnThread* nt) { + ++g_deliver_post_step_events_count; + nrn_deliver_events(nt); +} + +void spike_exchange_after_group(NrnThread* nt) { + if (!enabled() || !backend_native()) { + return; + } +#if NRNMPI + ++g_spike_exchange_count; + nrn_spike_exchange(nt); +#else + (void) nt; +#endif +} + +namespace detail { + +std::size_t deliver_net_events_count_for_testing() { + return g_deliver_net_events_count.load(); +} + +std::size_t deliver_post_step_events_count_for_testing() { + return g_deliver_post_step_events_count.load(); +} + +std::size_t spike_exchange_count_for_testing() { + return g_spike_exchange_count.load(); +} + +void reset_net_events_for_testing() { + g_deliver_net_events_count.store(0); + g_deliver_post_step_events_count.store(0); + g_spike_exchange_count.store(0); +} + +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/net_events.hpp b/src/neuron/gpu/net_events.hpp new file mode 100644 index 0000000000..c8c9c6ec9c --- /dev/null +++ b/src/neuron/gpu/net_events.hpp @@ -0,0 +1,29 @@ +#pragma once + +#include + +struct NrnThread; + +namespace neuron::gpu { + +/** + * Deliver presynaptic spikes from the CPU priority queue to NET_RECEIVE blocks. + * Cross-rank spikes arrive via MPI into per-rank queues on CPU; mechanism + * NET_RECEIVE computation itself runs on GPU during the integration step. + */ +void deliver_net_events_host(NrnThread* nt); + +/** Host-side POINT_PROCESS / NetCon event delivery after the integration half-step. */ +void deliver_post_step_events_host(NrnThread* nt); + +/** Host MPI spike exchange after a fixed-step group when native GPU is active. */ +void spike_exchange_after_group(NrnThread* nt); + +namespace detail { +[[nodiscard]] std::size_t deliver_net_events_count_for_testing(); +[[nodiscard]] std::size_t deliver_post_step_events_count_for_testing(); +[[nodiscard]] std::size_t spike_exchange_count_for_testing(); +void reset_net_events_for_testing(); +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/net_send_buffer.cpp b/src/neuron/gpu/net_send_buffer.cpp new file mode 100644 index 0000000000..dab6c36caa --- /dev/null +++ b/src/neuron/gpu/net_send_buffer.cpp @@ -0,0 +1,205 @@ +#include "neuron/gpu/net_send_buffer.hpp" + +#include "coreneuron/utils/offload.hpp" +#include "multicore.h" +#include "nrnoc_ml.h" +#include "section_fwd.hpp" + +#include +#include +#include +#include +#include + +extern void nrn_net_send(void* v, double* weight, Point_process* pnt, double td, double flag); +extern void net_event(Point_process* pnt, double time); +extern void nrn_net_move(Datum* v, Point_process* pnt, double tt); + +namespace neuron::gpu { + +std::vector net_buf_send_types; + +namespace { + +template +T* alloc_buffer(int size) { + return static_cast(std::calloc(static_cast(size), sizeof(T))); +} + +template +void grow_buffer(T** buf, int old_size, int new_size) { + T* new_buf = alloc_buffer(new_size); + if (*buf) { + std::copy_n(*buf, old_size, new_buf); + std::free(*buf); + } + *buf = new_buf; +} + +} // namespace + +NetSendBuffer_t::NetSendBuffer_t(int size) + : _size(size) { + _sendtype = alloc_buffer(_size); + _vdata_index = alloc_buffer(_size); + _pnt_index = alloc_buffer(_size); + _weight_index = alloc_buffer(_size); + _nsb_t = alloc_buffer(_size); + _nsb_flag = alloc_buffer(_size); + reallocated = 1; +} + +NetSendBuffer_t::~NetSendBuffer_t() { + std::free(_sendtype); + std::free(_vdata_index); + std::free(_pnt_index); + std::free(_weight_index); + std::free(_nsb_t); + std::free(_nsb_flag); +} + +void NetSendBuffer_t::grow() { +#if defined(NRN_ENABLE_GPU) + // GPU execution cannot reallocate on device; host grow is used before upload. + if (_cnt >= _size) { + int const new_size = std::max(_size * 2, _cnt + 1); + grow_buffer(&_sendtype, _size, new_size); + grow_buffer(&_vdata_index, _size, new_size); + grow_buffer(&_pnt_index, _size, new_size); + grow_buffer(&_weight_index, _size, new_size); + grow_buffer(&_nsb_t, _size, new_size); + grow_buffer(&_nsb_flag, _size, new_size); + _size = new_size; + reallocated = 1; + } +#else + (void) 0; +#endif +} + +std::size_t NetSendBuffer_t::size_of_object() const { + std::size_t nbytes = 0; + nbytes += static_cast(_size) * sizeof(int) * 4; + nbytes += static_cast(_size) * sizeof(double) * 2; + return nbytes; +} + +void net_send_buffer_ensure(Memb_list* ml) { + if (!ml || ml->_net_send_buffer) { + return; + } + int const capacity = std::max(8, ml->nodecount * 2); + ml->_net_send_buffer = new NetSendBuffer_t(capacity); +} + +void update_net_send_buffer_on_host(NrnThread* nt, NetSendBuffer_t* nsb) { +#if defined(NRN_ENABLE_GPU) + if (!nt || !nsb || !nt->compute_gpu) { + return; + } + if (nsb->_cnt > nsb->_size) { + fprintf(stderr, "ERROR: NetSendBuffer exceeded during GPU execution (thread %d)\n", nt->id); + std::abort(); + } + if (!nsb->_cnt) { + return; + } + // clang-format off + nrn_pragma_acc(update self(nsb->_sendtype[:nsb->_cnt], + nsb->_vdata_index[:nsb->_cnt], + nsb->_pnt_index[:nsb->_cnt], + nsb->_weight_index[:nsb->_cnt], + nsb->_nsb_t[:nsb->_cnt], + nsb->_nsb_flag[:nsb->_cnt]) + if (nsb->_cnt)) + nrn_pragma_omp(target update from(nsb->_sendtype[:nsb->_cnt], + nsb->_vdata_index[:nsb->_cnt], + nsb->_pnt_index[:nsb->_cnt], + nsb->_weight_index[:nsb->_cnt], + nsb->_nsb_t[:nsb->_cnt], + nsb->_nsb_flag[:nsb->_cnt]) + if (nsb->_cnt)) + // clang-format on +#else + (void) nt; + (void) nsb; +#endif +} + +void deliver_net_send_buffer_events(NrnThread* nt, NetSendBuffer_t* nsb) { + if (!nt || !nsb || !nsb->_cnt) { + return; + } + for (int i = 0; i < nsb->_cnt; ++i) { + auto* const pnt = reinterpret_cast( + static_cast(nsb->_pnt_index[i])); + auto* const weight = nsb->_weight_index[i] >= 0 + ? reinterpret_cast( + static_cast(nsb->_weight_index[i])) + : nullptr; + auto* const vdata = nsb->_vdata_index[i] >= 0 + ? reinterpret_cast( + static_cast(nsb->_vdata_index[i])) + : nullptr; + switch (nsb->_sendtype[i]) { + case 0: + nrn_net_send(vdata, weight, pnt, nsb->_nsb_t[i], nsb->_nsb_flag[i]); + break; + case 1: + net_event(pnt, nsb->_nsb_t[i]); + break; + case 2: + nrn_net_move(static_cast(vdata), pnt, nsb->_nsb_t[i]); + break; + default: + break; + } + } + nsb->_cnt = 0; +#if defined(NRN_ENABLE_GPU) + if (nt->compute_gpu) { + nrn_pragma_acc(update device(nsb->_cnt) if (nt->compute_gpu)) + nrn_pragma_omp(target update to(nsb->_cnt) if (nt->compute_gpu)) + } +#endif +} + +void ensure_thread_net_send_buffers(NrnThread* nt) { + if (!nt || !nt->_ml_list) { + return; + } + for (int type: net_buf_send_types) { + if (auto* ml = nt->_ml_list[type]) { + net_send_buffer_ensure(ml); + } + } + if (!nt->_net_send_buffer && nt->ncell > 0) { + nt->_net_send_buffer_size = std::max(8, nt->ncell); + nt->_net_send_buffer = alloc_buffer(nt->_net_send_buffer_size); + } +} + +void flush_mechanism_net_send_buffers(NrnThread* nt) { + if (!nt) { + return; + } + for (auto* tml = nt->tml; tml; tml = tml->next) { + if (auto* nsb = tml->ml->_net_send_buffer) { + update_net_send_buffer_on_host(nt, nsb); + deliver_net_send_buffer_events(nt, nsb); + } + } +} + +} // namespace neuron::gpu + +extern "C" void hoc_register_net_send_buffering(int type) { + if (type < 0) { + return; + } + if (std::find(neuron::gpu::net_buf_send_types.begin(), + neuron::gpu::net_buf_send_types.end(), + type) == neuron::gpu::net_buf_send_types.end()) { + neuron::gpu::net_buf_send_types.push_back(type); + } +} \ No newline at end of file diff --git a/src/neuron/gpu/net_send_buffer.hpp b/src/neuron/gpu/net_send_buffer.hpp new file mode 100644 index 0000000000..a65b31f40d --- /dev/null +++ b/src/neuron/gpu/net_send_buffer.hpp @@ -0,0 +1,45 @@ +#pragma once + +#include +#include + +struct Memb_list; +struct NrnThread; + +namespace neuron::gpu { + +/** + * Buffers net_send/net_event/net_move calls from GPU mechanism kernels until + * host flush. Cross-cell NetCon events go to the CPU spike system; self-events + * may remain on-device via the buffer (CoreNEURON-style). + */ +struct NetSendBuffer_t { + int* _sendtype = nullptr; + int* _vdata_index = nullptr; + int* _pnt_index = nullptr; + int* _weight_index = nullptr; + double* _nsb_t = nullptr; + double* _nsb_flag = nullptr; + int _cnt = 0; + int _size = 0; + int reallocated = 0; + + explicit NetSendBuffer_t(int size); + ~NetSendBuffer_t(); + + void grow(); + + [[nodiscard]] std::size_t size_of_object() const; +}; + +void net_send_buffer_ensure(Memb_list* ml); +void update_net_send_buffer_on_host(NrnThread* nt, NetSendBuffer_t* nsb); +void deliver_net_send_buffer_events(NrnThread* nt, NetSendBuffer_t* nsb); +void ensure_thread_net_send_buffers(NrnThread* nt); +void flush_mechanism_net_send_buffers(NrnThread* nt); + +extern std::vector net_buf_send_types; + +} // namespace neuron::gpu + +extern "C" void hoc_register_net_send_buffering(int type); \ No newline at end of file diff --git a/src/neuron/gpu/offload.cpp b/src/neuron/gpu/offload.cpp new file mode 100644 index 0000000000..63aa26b04d --- /dev/null +++ b/src/neuron/gpu/offload.cpp @@ -0,0 +1,242 @@ +/* +# ============================================================================= +# Copyright (c) 2016 - 2022 Blue Brain Project/EPFL +# +# See top-level LICENSE file for details. +# ============================================================================= +*/ + +#include "neuron/gpu/offload.hpp" + +#include +#include +#include +#include +#include + +#if defined(NRN_ENABLE_GPU) && defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENMP) && \ + __has_include() +#include +#endif + +#if __has_include() +#define NRN_GPU_USE_CXXABI +#include +#include +#endif + +#ifdef NRN_ENABLE_PRESENT_TABLE +#include +#include +#include +#include + +namespace { +struct present_table_value { + std::size_t ref_count{}, size{}; + std::byte* dev_ptr{}; +}; +std::map present_table; +std::shared_mutex present_table_mutex; +} // namespace +#endif + +namespace { +std::string cxx_demangle(const char* mangled) { +#ifdef NRN_GPU_USE_CXXABI + int status{}; + std::unique_ptr demangled{ + abi::__cxa_demangle(mangled, nullptr, nullptr, &status), free}; + return status ? mangled : demangled.get(); +#else + return mangled; +#endif +} + +bool target_debug_output_enabled() { + const char* env = std::getenv("NRN_GPU_DEBUG"); + if (!env) { + return false; + } + std::string env_s{env}; + if (env_s == "1") { + return true; + } + if (env_s == "0") { + return false; + } + throw std::runtime_error("NRN_GPU_DEBUG must be set to 0 or 1 (got " + env_s + ')'); +} + +bool target_enable_debug{target_debug_output_enabled()}; +} // namespace + +namespace neuron::gpu { + +void target_copyin_debug(std::string_view file, + int line, + std::size_t sizeof_T, + std::type_info const& typeid_T, + void const* h_ptr, + std::size_t len, + void* d_ptr) { + if (!target_enable_debug) { + return; + } + std::cerr << file << ':' << line << ": nrn_target_copyin<" << cxx_demangle(typeid_T.name()) + << ">(" << h_ptr << ", " << len << " * " << sizeof_T << " = " << len * sizeof_T + << ") -> " << d_ptr << std::endl; +} + +void target_delete_debug(std::string_view file, + int line, + std::size_t sizeof_T, + std::type_info const& typeid_T, + void const* h_ptr, + std::size_t len) { + if (!target_enable_debug) { + return; + } + std::cerr << file << ':' << line << ": nrn_target_delete<" << cxx_demangle(typeid_T.name()) + << ">(" << h_ptr << ", " << len << " * " << sizeof_T << " = " << len * sizeof_T << ')' + << std::endl; +} + +void target_deviceptr_debug(std::string_view file, + int line, + std::type_info const& typeid_T, + void const* h_ptr, + void* d_ptr) { + if (!target_enable_debug) { + return; + } + std::cerr << file << ':' << line << ": nrn_target_deviceptr<" << cxx_demangle(typeid_T.name()) + << ">(" << h_ptr << ") -> " << d_ptr << std::endl; +} + +void target_is_present_debug(std::string_view file, + int line, + std::type_info const& typeid_T, + void const* h_ptr, + void* d_ptr) { + if (!target_enable_debug) { + return; + } + std::cerr << file << ':' << line << ": nrn_target_is_present<" << cxx_demangle(typeid_T.name()) + << ">(" << h_ptr << ") -> " << d_ptr << std::endl; +} + +void target_memcpy_to_device_debug(std::string_view file, + int line, + std::size_t sizeof_T, + std::type_info const& typeid_T, + void const* h_ptr, + std::size_t len, + void* d_ptr) { + if (!target_enable_debug) { + return; + } + std::cerr << file << ':' << line << ": nrn_target_memcpy_to_device<" + << cxx_demangle(typeid_T.name()) << ">(" << d_ptr << ", " << h_ptr << ", " << len + << " * " << sizeof_T << " = " << len * sizeof_T << ')' << std::endl; +} + +#ifdef NRN_ENABLE_PRESENT_TABLE +std::pair target_deviceptr_impl(bool must_be_present_or_null, void const* h_ptr) { + if (!h_ptr) { + return {nullptr, false}; + } + std::shared_lock _{present_table_mutex}; + if (present_table.empty()) { + return {nullptr, must_be_present_or_null}; + } + auto const iter = std::prev(std::upper_bound( + present_table.begin(), present_table.end(), h_ptr, [](void const* hp, auto const& entry) { + return hp < entry.first; + })); + if (iter == present_table.end()) { + return {nullptr, must_be_present_or_null}; + } + std::byte const* const h_byte_ptr{static_cast(h_ptr)}; + std::byte const* const h_start_of_block{iter->first}; + std::size_t const block_size{iter->second.size}; + std::byte* const d_start_of_block{iter->second.dev_ptr}; + bool const is_present{h_byte_ptr < h_start_of_block + block_size}; + if (!is_present) { + return {nullptr, must_be_present_or_null}; + } + return {d_start_of_block + (h_byte_ptr - h_start_of_block), false}; +} + +void target_copyin_update_present_table(void const* h_ptr, void* d_ptr, std::size_t len) { + if (!h_ptr) { + assert(!d_ptr); + return; + } + std::lock_guard _{present_table_mutex}; + present_table_value new_val{}; + new_val.size = len; + new_val.ref_count = 1; + new_val.dev_ptr = static_cast(d_ptr); + auto const [iter, inserted] = present_table.emplace(static_cast(h_ptr), + std::move(new_val)); + if (!inserted) { + assert(iter->second.size == len); + assert(iter->second.dev_ptr == new_val.dev_ptr); + ++(iter->second.ref_count); + } +} + +void target_delete_update_present_table(void const* h_ptr, std::size_t len) { + if (!h_ptr) { + return; + } + std::lock_guard _{present_table_mutex}; + auto const iter = present_table.find(static_cast(h_ptr)); + assert(iter != present_table.end()); + assert(iter->second.size == len); + --(iter->second.ref_count); + if (iter->second.ref_count == 0) { + present_table.erase(iter); + } +} +#endif + +int target_get_num_devices() { +#if defined(NRN_ENABLE_GPU) && !defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENACC) + acc_device_t const device_type = acc_device_nvidia; + return acc_get_num_devices(device_type); +#elif defined(NRN_ENABLE_GPU) && defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENMP) + return omp_get_num_devices(); +#else + return 0; +#endif +} + +void target_set_default_device(int device_num) { +#if defined(NRN_ENABLE_GPU) && !defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENACC) + acc_set_device_num(device_num, acc_device_nvidia); +#elif defined(NRN_ENABLE_GPU) && defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENMP) + omp_set_default_device(device_num); +#if __has_include() + auto const cuda_code = cudaSetDevice(device_num); + if (cuda_code != cudaSuccess) { + throw std::runtime_error("neuron::gpu::target_set_default_device: cudaSetDevice failed"); + } +#endif +#else + (void) device_num; +#endif +} + +int target_get_default_device() { +#if defined(NRN_ENABLE_GPU) && !defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENACC) + return acc_get_device_num(acc_device_nvidia); +#elif defined(NRN_ENABLE_GPU) && defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENMP) + return omp_get_default_device(); +#else + return -1; +#endif +} + +} // namespace neuron::gpu \ No newline at end of file diff --git a/src/neuron/gpu/offload.hpp b/src/neuron/gpu/offload.hpp new file mode 100644 index 0000000000..c6d70e15f9 --- /dev/null +++ b/src/neuron/gpu/offload.hpp @@ -0,0 +1,201 @@ +/* +# ============================================================================= +# Copyright (c) 2016 - 2022 Blue Brain Project/EPFL +# +# See top-level LICENSE file for details. +# ============================================================================= +*/ +#pragma once + +// NEURON-native OpenACC/OpenMP offload helpers (forked from coreneuron/utils/offload.hpp). +// CoreNEURON continues to use coreneuron::cnrn_target_* during the Phase A transition. + +#define nrn_gpu_pragma_stringify(x) #x +#if defined(NRN_ENABLE_GPU) && defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENMP) +#define nrn_gpu_pragma_acc(x) +#define nrn_gpu_pragma_omp(x) _Pragma(nrn_gpu_pragma_stringify(omp x)) +#include +#elif defined(NRN_ENABLE_GPU) && !defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENACC) +#define nrn_gpu_pragma_acc(x) _Pragma(nrn_gpu_pragma_stringify(acc x)) +#define nrn_gpu_pragma_omp(x) +#include +#else +#define nrn_gpu_pragma_acc(x) +#define nrn_gpu_pragma_omp(x) +#endif + +// nrnoc GPU paths (e.g. treeset.cpp) use the same names as CoreNEURON offload.hpp. +#define nrn_pragma_acc(x) nrn_gpu_pragma_acc(x) +#define nrn_pragma_omp(x) nrn_gpu_pragma_omp(x) + +#include +#include +#include +#include +#include + +namespace neuron::gpu { + +int target_get_num_devices(); +void target_set_default_device(int device_num); +int target_get_default_device(); + +void target_copyin_debug(std::string_view file, + int line, + std::size_t sizeof_T, + std::type_info const& typeid_T, + void const* h_ptr, + std::size_t len, + void* d_ptr); +void target_delete_debug(std::string_view file, + int line, + std::size_t sizeof_T, + std::type_info const& typeid_T, + void const* h_ptr, + std::size_t len); +void target_deviceptr_debug(std::string_view file, + int line, + std::type_info const& typeid_T, + void const* h_ptr, + void* d_ptr); +void target_is_present_debug(std::string_view file, + int line, + std::type_info const& typeid_T, + void const* h_ptr, + void* d_ptr); +void target_memcpy_to_device_debug(std::string_view file, + int line, + std::size_t sizeof_T, + std::type_info const& typeid_T, + void const* h_ptr, + std::size_t len, + void* d_ptr); + +#if defined(NRN_ENABLE_GPU) && !defined(NRN_UNIFIED_MEMORY) && defined(__NVCOMPILER_MAJOR__) && \ + defined(__NVCOMPILER_MINOR__) && (__NVCOMPILER_MAJOR__ <= 22) && (__NVCOMPILER_MINOR__ <= 3) +// Homegrown implementation for buggy NVHPC versions (<=22.3); required for dynamically loaded +// mechanisms. See NVIDIA forum thread 211599. +#define NRN_ENABLE_PRESENT_TABLE +std::pair target_deviceptr_impl(bool must_be_present_or_null, void const* h_ptr); +void target_copyin_update_present_table(void const* h_ptr, void* d_ptr, std::size_t len); +void target_delete_update_present_table(void const* h_ptr, std::size_t len); +#endif + +template +T* target_deviceptr_or_present(std::string_view file, + int line, + bool must_be_present_or_null, + const T* h_ptr) { + T* d_ptr{}; + bool error{false}; +#ifdef NRN_ENABLE_PRESENT_TABLE + auto const d_ptr_and_error = target_deviceptr_impl(must_be_present_or_null, h_ptr); + d_ptr = static_cast(d_ptr_and_error.first); + error = d_ptr_and_error.second; +#elif defined(NRN_ENABLE_GPU) && !defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENACC) + d_ptr = static_cast(acc_deviceptr(const_cast(h_ptr))); +#elif defined(NRN_ENABLE_GPU) && defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENMP) + if (must_be_present_or_null || omp_target_is_present(h_ptr, omp_get_default_device())) { + nrn_gpu_pragma_omp(target data use_device_ptr(h_ptr)) { + d_ptr = const_cast(h_ptr); + } + } +#else + if (must_be_present_or_null && h_ptr) { + throw std::runtime_error( + "neuron::gpu::target_deviceptr() not implemented without OpenACC/OpenMP and " + "NRN_ENABLE_GPU"); + } +#endif + if (must_be_present_or_null) { + target_deviceptr_debug(file, line, typeid(T), h_ptr, d_ptr); + } else { + target_is_present_debug(file, line, typeid(T), h_ptr, d_ptr); + } + if (error) { + throw std::runtime_error( + "neuron::gpu::target_deviceptr() encountered an error; try NRN_GPU_DEBUG=1"); + } + return d_ptr; +} + +template +T* target_copyin(std::string_view file, int line, const T* h_ptr, std::size_t len = 1) { + T* d_ptr{}; +#if defined(NRN_ENABLE_GPU) && !defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENACC) + d_ptr = static_cast(acc_copyin(const_cast(h_ptr), len * sizeof(T))); +#elif defined(NRN_ENABLE_GPU) && defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENMP) + nrn_gpu_pragma_omp(target enter data map(to + : h_ptr[:len])) + nrn_gpu_pragma_omp(target data use_device_ptr(h_ptr)) { + d_ptr = const_cast(h_ptr); + } +#else + throw std::runtime_error( + "neuron::gpu::target_copyin() not implemented without OpenACC/OpenMP and NRN_ENABLE_GPU"); +#endif +#ifdef NRN_ENABLE_PRESENT_TABLE + target_copyin_update_present_table(h_ptr, d_ptr, len * sizeof(T)); +#endif + target_copyin_debug(file, line, sizeof(T), typeid(T), h_ptr, len, d_ptr); + return d_ptr; +} + +template +void target_delete(std::string_view file, int line, T* h_ptr, std::size_t len = 1) { + target_delete_debug(file, line, sizeof(T), typeid(T), h_ptr, len); +#ifdef NRN_ENABLE_PRESENT_TABLE + target_delete_update_present_table(h_ptr, len * sizeof(T)); +#endif +#if defined(NRN_ENABLE_GPU) && !defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENACC) + acc_delete(h_ptr, len * sizeof(T)); +#elif defined(NRN_ENABLE_GPU) && defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENMP) + nrn_gpu_pragma_omp(target exit data map(delete : h_ptr[:len])) +#else + throw std::runtime_error( + "neuron::gpu::target_delete() not implemented without OpenACC/OpenMP and NRN_ENABLE_GPU"); +#endif +} + +template +void target_memcpy_to_device(std::string_view file, + int line, + T* d_ptr, + const T* h_ptr, + std::size_t len = 1) { + target_memcpy_to_device_debug(file, line, sizeof(T), typeid(T), h_ptr, len, d_ptr); +#if defined(NRN_ENABLE_GPU) && !defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENACC) + acc_memcpy_to_device(d_ptr, const_cast(h_ptr), len * sizeof(T)); +#elif defined(NRN_ENABLE_GPU) && defined(NRN_PREFER_OPENMP_OFFLOAD) && defined(_OPENMP) + omp_target_memcpy(d_ptr, + const_cast(h_ptr), + len * sizeof(T), + 0, + 0, + omp_get_default_device(), + omp_get_initial_device()); +#else + throw std::runtime_error( + "neuron::gpu::target_memcpy_to_device() not implemented without OpenACC/OpenMP and " + "NRN_ENABLE_GPU"); +#endif +} + +template +void target_update_on_device(std::string_view file, int line, const T* h_ptr, std::size_t len = 1) { + auto* d_ptr = target_deviceptr_or_present(file, line, true, h_ptr); + target_memcpy_to_device(file, line, d_ptr, h_ptr, len); +} + +} // namespace neuron::gpu + +#define nrn_target_copyin(...) neuron::gpu::target_copyin(__FILE__, __LINE__, __VA_ARGS__) +#define nrn_target_delete(...) neuron::gpu::target_delete(__FILE__, __LINE__, __VA_ARGS__) +#define nrn_target_is_present(...) \ + neuron::gpu::target_deviceptr_or_present(__FILE__, __LINE__, false, __VA_ARGS__) +#define nrn_target_deviceptr(...) \ + neuron::gpu::target_deviceptr_or_present(__FILE__, __LINE__, true, __VA_ARGS__) +#define nrn_target_memcpy_to_device(...) \ + neuron::gpu::target_memcpy_to_device(__FILE__, __LINE__, __VA_ARGS__) +#define nrn_target_update_on_device(...) \ + neuron::gpu::target_update_on_device(__FILE__, __LINE__, __VA_ARGS__) \ No newline at end of file diff --git a/src/neuron/gpu/post_solve.cpp b/src/neuron/gpu/post_solve.cpp new file mode 100644 index 0000000000..1da09a9c87 --- /dev/null +++ b/src/neuron/gpu/post_solve.cpp @@ -0,0 +1,169 @@ +#include "neuron/gpu/post_solve.hpp" + +#include "coreneuron/utils/offload.hpp" +#include "membfunc.h" +#include "multicore.h" +#include "neuron/cache/mechanism_range.hpp" +#include "neuron/model_data.hpp" +#include "nrn_ansi.h" +#include "nrnoc_ml.h" + +extern int secondorder; + +extern void (*nrnthread_vi_compute_)(NrnThread*); + +namespace neuron::gpu { +namespace { + +constexpr int cap_cm_index = 0; +constexpr int cap_i_cap_index = 1; +constexpr int ion_cur_index = 3; +constexpr int ion_dcurdv_index = 4; + +void second_order_cur_on_device(model_sorted_token const& sorted_token, NrnThread& nt) { + if (secondorder != 2 || nt.end <= 0) { + return; + } + auto* const vec_rhs = nt.node_rhs_storage(); + for (auto* tml = nt.tml; tml; tml = tml->next) { + if (!nrn_is_ion(tml->index)) { + continue; + } + auto* const ml = tml->ml; + int const count = ml->nodecount; + if (count <= 0) { + continue; + } + neuron::cache::MechanismRange<5, 1> ml_cache{sorted_token, *ml}; + double* const cur = ml_cache.data_array_ptr(); + double* const dcurdv = ml_cache.data_array_ptr(); + int* const ni = ml->nodeindices; + // clang-format off + nrn_pragma_acc(parallel loop present(cur [0:count], + dcurdv [0:count], + ni [0:count], + vec_rhs [0:nt.end]) if (nt.compute_gpu) + async(nt.stream_id)) + // clang-format on + nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu)) + for (int i = 0; i < count; ++i) { + cur[i] += dcurdv[i] * vec_rhs[ni[i]]; + } + } +} + +void update_voltage_on_device(NrnThread& nt) { + if (nt.end <= 0) { + return; + } + auto* const vec_rhs = nt.node_rhs_storage(); + auto* const vec_v = nt.node_voltage_storage(); + if (secondorder) { + // clang-format off + nrn_pragma_acc(parallel loop present(vec_v [0:nt.end], vec_rhs [0:nt.end]) if (nt.compute_gpu) + async(nt.stream_id)) + // clang-format on + nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu)) + for (int i = 0; i < nt.end; ++i) { + vec_v[i] += 2. * vec_rhs[i]; + } + } else { + // clang-format off + nrn_pragma_acc(parallel loop present(vec_v [0:nt.end], vec_rhs [0:nt.end]) if (nt.compute_gpu) + async(nt.stream_id)) + // clang-format on + nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu)) + for (int i = 0; i < nt.end; ++i) { + vec_v[i] += vec_rhs[i]; + } + } +} + +void capacity_current_on_device(model_sorted_token const& sorted_token, NrnThread& nt) { +#if I_MEMBRANE + if (!nt.tml || nt.tml->index != CAP || nt.end <= 0) { + return; + } + auto* const ml = nt.tml->ml; + int const count = ml->nodecount; + if (count <= 0) { + return; + } + neuron::cache::MechanismRange<2, 0> ml_cache{sorted_token, *ml}; + auto* const vec_rhs = nt.node_rhs_storage(); + double* const cm = ml_cache.data_array_ptr(); + double* const i_cap = ml_cache.data_array_ptr(); + int* const ni = ml->nodeindices; + double const cfac = .001 * nt.cj; + // clang-format off + nrn_pragma_acc(parallel loop present(vec_rhs [0:nt.end], + cm [0:count], + i_cap [0:count], + ni [0:count]) if (nt.compute_gpu) + async(nt.stream_id)) + // clang-format on + nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu)) + for (int i = 0; i < count; ++i) { + i_cap[i] = cfac * cm[i] * vec_rhs[ni[i]]; + } +#else + (void) sorted_token; + (void) nt; +#endif +} + +void fast_imem_on_device(NrnThread& nt) { + if (!::nrn_use_fast_imem || nt.end <= 0) { + return; + } + auto* const vec_area = nt.node_area_storage(); + auto* const vec_rhs = nt.node_rhs_storage(); + auto* const vec_sav_d = nt.node_sav_d_storage(); + auto* const vec_sav_rhs = nt.node_sav_rhs_storage(); + if (!vec_sav_d || !vec_sav_rhs) { + return; + } + // clang-format off + nrn_pragma_acc(parallel loop present(vec_rhs [0:nt.end], + vec_area [0:nt.end], + vec_sav_d [0:nt.end], + vec_sav_rhs [0:nt.end]) if (nt.compute_gpu) + async(nt.stream_id)) + // clang-format on + nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu)) + for (int i = 0; i < nt.end; ++i) { + vec_sav_rhs[i] = (vec_sav_d[i] * vec_rhs[i] + vec_sav_rhs[i]) * vec_area[i] * 0.01; + } +} + +} // namespace + +bool post_solve_needs_host_fallback(NrnThread const& nt) { + extern int use_sparse13; + if (use_sparse13) { + return true; + } +#if EXTRACELLULAR + (void) nt; + return true; +#else + return nrnthread_vi_compute_ != nullptr; +#endif +} + +void post_solve_on_device(model_sorted_token const& sorted_token, NrnThread& nt) { + if (!nt.compute_gpu || nt.end <= 0) { + return; + } + + second_order_cur_on_device(sorted_token, nt); + update_voltage_on_device(nt); + capacity_current_on_device(sorted_token, nt); + fast_imem_on_device(nt); + +#if defined(NRN_ENABLE_GPU) + nrn_pragma_acc(wait(nt.stream_id)) +#endif +} + +} // namespace neuron::gpu \ No newline at end of file diff --git a/src/neuron/gpu/post_solve.hpp b/src/neuron/gpu/post_solve.hpp new file mode 100644 index 0000000000..6cd265645a --- /dev/null +++ b/src/neuron/gpu/post_solve.hpp @@ -0,0 +1,17 @@ +#pragma once + +struct NrnThread; + +namespace neuron { +struct model_sorted_token; +} + +namespace neuron::gpu { + +/** True when post-solve must run on the host (sparse13, extracellular, LFP hooks). */ +[[nodiscard]] bool post_solve_needs_host_fallback(NrnThread const& nt); + +/** GPU second-order ion correction, voltage update, capacity current, and fast_imem. */ +void post_solve_on_device(model_sorted_token const& sorted_token, NrnThread& nt); + +} // namespace neuron::gpu \ No newline at end of file diff --git a/src/neuron/gpu/sync.cpp b/src/neuron/gpu/sync.cpp new file mode 100644 index 0000000000..a455572383 --- /dev/null +++ b/src/neuron/gpu/sync.cpp @@ -0,0 +1,175 @@ +#include "neuron/gpu/sync.hpp" + +#include "coreneuron/utils/offload.hpp" +#include "multicore.h" +#include "nrn_ansi.h" + +namespace neuron::gpu { +namespace { + +void sync_node_voltages_to_host(NrnThread& nt) { +#if defined(NRN_ENABLE_GPU) + if (!nt.compute_gpu || nt.end <= 0) { + return; + } + auto* const vec_v = nt.node_voltage_storage(); + nrn_pragma_acc(update host(vec_v [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id)) + nrn_pragma_omp(target update from(vec_v [0:nt.end]) if (nt.compute_gpu)) + nrn_pragma_acc(wait(nt.stream_id)) +#else + (void) nt; +#endif +} + +void sync_node_voltages_to_device(NrnThread& nt) { +#if defined(NRN_ENABLE_GPU) + if (!nt.compute_gpu || nt.end <= 0) { + return; + } + auto* const vec_v = nt.node_voltage_storage(); + nrn_pragma_acc(update device(vec_v [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id)) + nrn_pragma_omp(target update to(vec_v [0:nt.end]) if (nt.compute_gpu)) + nrn_pragma_acc(wait(nt.stream_id)) +#else + (void) nt; +#endif +} + +void sync_matrix_arrays_to_device(NrnThread& nt) { +#if defined(NRN_ENABLE_GPU) + if (!nt.compute_gpu || nt.end <= 0) { + return; + } + auto* const vec_rhs = nt.node_rhs_storage(); + auto* const vec_d = nt.node_d_storage(); + nrn_pragma_acc(update device(vec_rhs [0:nt.end], vec_d [0:nt.end]) if (nt.compute_gpu) + async(nt.stream_id)) + nrn_pragma_omp(target update to(vec_rhs [0:nt.end], vec_d [0:nt.end]) if (nt.compute_gpu)) + if (auto* const vec_sav_rhs = nt.node_sav_rhs_storage()) { + nrn_pragma_acc(update device(vec_sav_rhs [0:nt.end]) if (nt.compute_gpu) + async(nt.stream_id)) + nrn_pragma_omp(target update to(vec_sav_rhs [0:nt.end]) if (nt.compute_gpu)) + } + if (auto* const vec_sav_d = nt.node_sav_d_storage()) { + nrn_pragma_acc(update device(vec_sav_d [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id)) + nrn_pragma_omp(target update to(vec_sav_d [0:nt.end]) if (nt.compute_gpu)) + } + nrn_pragma_acc(wait(nt.stream_id)) +#else + (void) nt; +#endif +} + +void sync_matrix_arrays_to_host(NrnThread& nt) { +#if defined(NRN_ENABLE_GPU) + if (!nt.compute_gpu || nt.end <= 0) { + return; + } + auto* const vec_rhs = nt.node_rhs_storage(); + auto* const vec_d = nt.node_d_storage(); + nrn_pragma_acc(update host(vec_rhs [0:nt.end], vec_d [0:nt.end]) if (nt.compute_gpu) + async(nt.stream_id)) + nrn_pragma_omp(target update from(vec_rhs [0:nt.end], vec_d [0:nt.end]) if (nt.compute_gpu)) + if (auto* const vec_sav_rhs = nt.node_sav_rhs_storage()) { + nrn_pragma_acc(update host(vec_sav_rhs [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id)) + nrn_pragma_omp(target update from(vec_sav_rhs [0:nt.end]) if (nt.compute_gpu)) + } + if (auto* const vec_sav_d = nt.node_sav_d_storage()) { + nrn_pragma_acc(update host(vec_sav_d [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id)) + nrn_pragma_omp(target update from(vec_sav_d [0:nt.end]) if (nt.compute_gpu)) + } + nrn_pragma_acc(wait(nt.stream_id)) +#else + (void) nt; +#endif +} + +} // namespace + +void sync_before_vecplay(NrnThread& nt) { + sync_node_voltages_to_host(nt); +} + +void sync_after_vecplay(NrnThread& nt) { + sync_node_voltages_to_device(nt); +} + +void sync_voltages_to_device_before_axial(NrnThread& nt) { + sync_node_voltages_to_device(nt); +} + +void sync_matrix_to_device_after_mechanisms(NrnThread& nt) { + sync_matrix_arrays_to_device(nt); +} + +void sync_diagonal_to_device_after_mechanisms(NrnThread& nt) { +#if defined(NRN_ENABLE_GPU) + if (!nt.compute_gpu || nt.end <= 0) { + return; + } + auto* const vec_d = nt.node_d_storage(); + nrn_pragma_acc(update device(vec_d [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id)) + nrn_pragma_omp(target update to(vec_d [0:nt.end]) if (nt.compute_gpu)) + if (auto* const vec_sav_d = nt.node_sav_d_storage()) { + nrn_pragma_acc(update device(vec_sav_d [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id)) + nrn_pragma_omp(target update to(vec_sav_d [0:nt.end]) if (nt.compute_gpu)) + } + nrn_pragma_acc(wait(nt.stream_id)) +#else + (void) nt; +#endif +} + +void sync_matrix_to_host_before_solve(NrnThread& nt) { + sync_matrix_arrays_to_host(nt); +} + +void sync_matrix_to_device_before_solve(NrnThread& nt) { + sync_matrix_arrays_to_device(nt); +} + +void sync_rhs_to_host_after_solve(NrnThread& nt) { +#if defined(NRN_ENABLE_GPU) + if (!nt.compute_gpu || nt.end <= 0) { + return; + } + auto* const vec_rhs = nt.node_rhs_storage(); + nrn_pragma_acc(update host(vec_rhs [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id)) + nrn_pragma_omp(target update from(vec_rhs [0:nt.end]) if (nt.compute_gpu)) + nrn_pragma_acc(wait(nt.stream_id)) +#else + (void) nt; +#endif +} + +void sync_voltages_to_host_after_post_solve(NrnThread& nt) { + sync_node_voltages_to_host(nt); +} + +void sync_fast_imem_to_host_after_post_solve(NrnThread& nt) { +#if defined(NRN_ENABLE_GPU) + if (!nt.compute_gpu || nt.end <= 0) { + return; + } + if (!::nrn_use_fast_imem) { + return; + } + if (auto* const vec_sav_rhs = nt.node_sav_rhs_storage()) { + nrn_pragma_acc(update host(vec_sav_rhs [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id)) + nrn_pragma_omp(target update from(vec_sav_rhs [0:nt.end]) if (nt.compute_gpu)) + nrn_pragma_acc(wait(nt.stream_id)) + } +#else + (void) nt; +#endif +} + +void sync_gap_after_voltage_update(NrnThread& nt) { + sync_node_voltages_to_host(nt); +} + +void sync_gap_after_host_voltage_update(NrnThread& nt) { + sync_node_voltages_to_device(nt); +} + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/sync.hpp b/src/neuron/gpu/sync.hpp new file mode 100644 index 0000000000..d04059a29d --- /dev/null +++ b/src/neuron/gpu/sync.hpp @@ -0,0 +1,43 @@ +#pragma once + +struct NrnThread; + +namespace neuron::gpu { + +/** Pull node voltages to host before host-side VecPlay / stimulus updates. */ +void sync_before_vecplay(NrnThread& nt); + +/** Push node voltages back to device after VecPlay updates. */ +void sync_after_vecplay(NrnThread& nt); + +/** Push node voltages to device immediately before OpenACC axial matrix assembly. */ +void sync_voltages_to_device_before_axial(NrnThread& nt); + +/** Push mechanism-updated matrix state to device before OpenACC axial loops. */ +void sync_matrix_to_device_after_mechanisms(NrnThread& nt); + +/** Push vec_d / sav_d only (preserve device vec_rhs after rhs axial). */ +void sync_diagonal_to_device_after_mechanisms(NrnThread& nt); + +/** Pull GPU axial matrix state to host before host nonvint adjustments. */ +void sync_matrix_to_host_before_solve(NrnThread& nt); + +/** Push post-nonvint matrix state to device before the GPU Hines solver. */ +void sync_matrix_to_device_before_solve(NrnThread& nt); + +/** Pull vec_rhs solution to host after the GPU solver (host post-solve fallback only). */ +void sync_rhs_to_host_after_solve(NrnThread& nt); + +/** Pull post-solve node voltages to host for HOC reads and VecPlay. */ +void sync_voltages_to_host_after_post_solve(NrnThread& nt); + +/** Pull fast_imem sav_rhs to host after GPU fast_imem (smaller than vec_rhs sync). */ +void sync_fast_imem_to_host_after_post_solve(NrnThread& nt); + +/** Pull device post-solve voltages to host before gap gather (device post-solve path). */ +void sync_gap_after_voltage_update(NrnThread& nt); + +/** Push host post-solve voltages to device before gap gather (host fallback path). */ +void sync_gap_after_host_voltage_update(NrnThread& nt); + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/upload.cpp b/src/neuron/gpu/upload.cpp new file mode 100644 index 0000000000..036072d012 --- /dev/null +++ b/src/neuron/gpu/upload.cpp @@ -0,0 +1,249 @@ +#include "neuron/gpu/upload.hpp" + +#include "coreneuron/permute/cellorder.hpp" +#include "multicore.h" +#include "neuron/container/mechanism_data.hpp" +#include "neuron/container/node_data.hpp" +#include "neuron/gpu/net_send_buffer.hpp" +#include "neuron/gpu/offload.hpp" +#include "neuron/model_data.hpp" +#include "node_order_optim/node_order_optim.h" +#include "nrnoc_ml.h" + +#include + +extern int nrn_nthread; + +namespace neuron { +extern int interleave_permute_type; +extern InterleaveInfo* interleave_info; +} // namespace neuron + +namespace neuron::gpu { +namespace { + +template +void copyin_pod_array(T const* host, std::size_t count, UploadState& state) { + if (!host || count == 0) { + return; + } +#if defined(NRN_ENABLE_GPU) && defined(_OPENACC) + nrn_target_copyin(host, count); + state.record(host, count, sizeof(T)); +#else + (void) state; + throw std::runtime_error("neuron::gpu::upload requires NRN_ENABLE_GPU with OpenACC"); +#endif +} + +template +void upload_soa_storage(Storage const& storage, UploadState& state) { + storage.for_each_vector_for_gpu_upload( + [&](auto const& /*tag*/, auto const& vec, int /*field_index*/, int /*array_dim*/) { + if (vec.empty()) { + return; + } + copyin_pod_array(vec.data(), vec.size(), state); + }); +} + +void upload_interleave_info_type1(InterleaveInfo& info, int ncell, UploadState& state) { + if (!info.stride || !info.firstnode || !info.lastnode || !info.cellsize || ncell <= 0) { + return; + } + + InterleaveInfo* d_info = nrn_target_copyin(&info, 1); + state.record(&info, 1, sizeof(InterleaveInfo)); + + int* d_stride = nrn_target_copyin(info.stride, static_cast(info.nstride + 1)); + state.record(info.stride, static_cast(info.nstride + 1), sizeof(int)); + nrn_target_memcpy_to_device(&(d_info->stride), &d_stride, 1); + + auto const ncell_sz = static_cast(ncell); + int* d_firstnode = nrn_target_copyin(info.firstnode, ncell_sz); + state.record(info.firstnode, ncell_sz, sizeof(int)); + nrn_target_memcpy_to_device(&(d_info->firstnode), &d_firstnode, 1); + + int* d_lastnode = nrn_target_copyin(info.lastnode, ncell_sz); + state.record(info.lastnode, ncell_sz, sizeof(int)); + nrn_target_memcpy_to_device(&(d_info->lastnode), &d_lastnode, 1); + + int* d_cellsize = nrn_target_copyin(info.cellsize, ncell_sz); + state.record(info.cellsize, ncell_sz, sizeof(int)); + nrn_target_memcpy_to_device(&(d_info->cellsize), &d_cellsize, 1); +} + +void upload_interleave_info_type2(InterleaveInfo& info, UploadState& state) { + if (!info.stride || !info.firstnode || !info.lastnode || !info.stridedispl || !info.cellsize) { + return; + } + if (info.nwarp <= 0) { + return; + } + + InterleaveInfo* d_info = nrn_target_copyin(&info, 1); + state.record(&info, 1, sizeof(InterleaveInfo)); + + auto const nstride = static_cast(info.nstride); + int* d_stride = nrn_target_copyin(info.stride, nstride); + state.record(info.stride, nstride, sizeof(int)); + nrn_target_memcpy_to_device(&(d_info->stride), &d_stride, 1); + + auto const nwarp_p1 = static_cast(info.nwarp + 1); + int* d_firstnode = nrn_target_copyin(info.firstnode, nwarp_p1); + state.record(info.firstnode, nwarp_p1, sizeof(int)); + nrn_target_memcpy_to_device(&(d_info->firstnode), &d_firstnode, 1); + + int* d_lastnode = nrn_target_copyin(info.lastnode, nwarp_p1); + state.record(info.lastnode, nwarp_p1, sizeof(int)); + nrn_target_memcpy_to_device(&(d_info->lastnode), &d_lastnode, 1); + + int* d_stridedispl = nrn_target_copyin(info.stridedispl, nwarp_p1); + state.record(info.stridedispl, nwarp_p1, sizeof(int)); + nrn_target_memcpy_to_device(&(d_info->stridedispl), &d_stridedispl, 1); + + auto const nwarp = static_cast(info.nwarp); + int* d_cellsize = nrn_target_copyin(info.cellsize, nwarp); + state.record(info.cellsize, nwarp, sizeof(int)); + nrn_target_memcpy_to_device(&(d_info->cellsize), &d_cellsize, 1); +} + +void upload_interleave_infos(UploadState& state) { + if (!interleave_info || interleave_permute_type == 0) { + return; + } + for (int ith = 0; ith < nrn_nthread; ++ith) { + auto& info = interleave_info[ith]; + if (interleave_permute_type == 2) { + upload_interleave_info_type2(info, state); + } else if (interleave_permute_type == 1) { + upload_interleave_info_type1(info, interleave_ncell_for_thread(ith), state); + } + } +} + +void upload_thread_parent_indices(UploadState& state) { + for (int ith = 0; ith < nrn_nthread; ++ith) { + auto& nt = nrn_threads[ith]; + if (!nt._v_parent_index || nt.end <= 0) { + continue; + } + copyin_pod_array(nt._v_parent_index, static_cast(nt.end), state); + } +} + +void upload_mechanism_nodeindices(UploadState& state) { + for (int ith = 0; ith < nrn_nthread; ++ith) { + auto& nt = nrn_threads[ith]; + for (auto* tml = nt.tml; tml; tml = tml->next) { + auto* const ml = tml->ml; + if (!ml || !ml->nodeindices || ml->nodecount <= 0) { + continue; + } + copyin_pod_array(ml->nodeindices, static_cast(ml->nodecount), state); + } + } +} + +void upload_nrnthread_shells(UploadState& state) { + for (int ith = 0; ith < nrn_nthread; ++ith) { + auto& nt = nrn_threads[ith]; + if (nt.end <= 0) { + continue; + } + auto* d_nt = nrn_target_copyin(&nt, 1); + state.record(&nt, 1, sizeof(NrnThread)); + if (nt._v_parent_index) { + int* d_parent = nrn_target_deviceptr(nt._v_parent_index); + nrn_target_memcpy_to_device(&(d_nt->_v_parent_index), &d_parent, 1); + } + } +} + +} // namespace + +void UploadState::record(void const* host, std::size_t count, std::size_t sizeof_elem) { + mirrors_.push_back({host, count, sizeof_elem}); +} + +bool UploadState::is_present(void const* host_ptr) const { +#if defined(NRN_ENABLE_GPU) && defined(_OPENACC) + if (!host_ptr) { + return false; + } + return nrn_target_is_present(host_ptr) != nullptr; +#else + (void) host_ptr; + return false; +#endif +} + +void UploadState::teardown() { +#if defined(NRN_ENABLE_GPU) && defined(_OPENACC) + for (auto const& mirror: mirrors_) { + if (!mirror.host || mirror.count == 0) { + continue; + } + if (mirror.sizeof_elem == sizeof(int)) { + nrn_target_delete(const_cast(static_cast(mirror.host)), mirror.count); + } else if (mirror.sizeof_elem == sizeof(double)) { + nrn_target_delete(const_cast(static_cast(mirror.host)), + mirror.count); + } else if (mirror.sizeof_elem == sizeof(InterleaveInfo)) { + nrn_target_delete(const_cast( + static_cast(mirror.host)), + mirror.count); + } else if (mirror.sizeof_elem == sizeof(NrnThread)) { + nrn_target_delete(const_cast(static_cast(mirror.host)), + mirror.count); + } else if (mirror.sizeof_elem == sizeof(Memb_list)) { + nrn_target_delete(const_cast(static_cast(mirror.host)), + mirror.count); + } else if (mirror.sizeof_elem == sizeof(Datum)) { + nrn_target_delete(const_cast(static_cast(mirror.host)), + mirror.count); + } else if (mirror.sizeof_elem == sizeof(Datum*)) { + nrn_target_delete(const_cast(static_cast(mirror.host)), + mirror.count); + } else if (mirror.sizeof_elem == sizeof(Memb_list*)) { + nrn_target_delete(const_cast(static_cast(mirror.host)), + mirror.count); + } else if (mirror.sizeof_elem == sizeof(NetSendBuffer_t)) { + nrn_target_delete(const_cast( + static_cast(mirror.host)), + mirror.count); + } + } +#endif + mirrors_.clear(); +} + +void upload_sorted_model(model_sorted_token const& sorted, UploadState& state) { + upload_soa_storage(sorted.node_data_token.container(), state); + + neuron::model().apply_to_mechanisms( + [&](auto& mech_data) { upload_soa_storage(mech_data, state); }); + + upload_thread_parent_indices(state); + upload_mechanism_nodeindices(state); + upload_nrnthread_shells(state); + upload_mechanism_lists(state); + upload_interleave_infos(state); +} + +namespace detail { + +void upload_interleave_info_for_testing(int permute_type, + InterleaveInfo& info, + int ncell, + UploadState& state) { + if (permute_type == 2) { + upload_interleave_info_type2(info, state); + } else if (permute_type == 1) { + upload_interleave_info_type1(info, ncell, state); + } +} + +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/upload.hpp b/src/neuron/gpu/upload.hpp new file mode 100644 index 0000000000..e17137b9a2 --- /dev/null +++ b/src/neuron/gpu/upload.hpp @@ -0,0 +1,50 @@ +#pragma once + +#include +#include + +namespace neuron { +struct model_sorted_token; +class InterleaveInfo; +} // namespace neuron + +namespace neuron::gpu { + +/** Tracks host buffers mirrored on the device for teardown via nrn_target_delete. */ +class UploadState { + public: + void teardown(); + + [[nodiscard]] std::size_t mirror_count() const noexcept { + return mirrors_.size(); + } + + [[nodiscard]] bool is_present(void const* host_ptr) const; + + void record(void const* host, std::size_t count, std::size_t sizeof_elem); + + private: + struct Mirror { + void const* host{}; + std::size_t count{}; + std::size_t sizeof_elem{}; + }; + + std::vector mirrors_{}; +}; + +/** Upload sorted node/mech SOA vectors and InterleaveInfo (permute 1/2) to the device. */ +void upload_sorted_model(model_sorted_token const& sorted, UploadState& state); + +/** Upload Memb_list shells, padded pdata, and per-thread _ml_list pointer arrays. */ +void upload_mechanism_lists(UploadState& state); + +namespace detail { +/** Test hook: upload one InterleaveInfo with CoreNEURON struct-then-patch pattern. */ +void upload_interleave_info_for_testing(int permute_type, + InterleaveInfo& info, + int ncell, + UploadState& state); +} // namespace detail + +} // namespace neuron::gpu diff --git a/src/neuron/gpu/upload_mechanisms.cpp b/src/neuron/gpu/upload_mechanisms.cpp new file mode 100644 index 0000000000..621f9b29d4 --- /dev/null +++ b/src/neuron/gpu/upload_mechanisms.cpp @@ -0,0 +1,151 @@ +#include "neuron/gpu/upload.hpp" + +#include "coreneuron/permute/data_layout.hpp" +#include "membfunc.h" +#include "multicore.h" +#include "neuron/gpu/net_send_buffer.hpp" +#include "neuron/gpu/offload.hpp" +#include "nrnoc_ml.h" + +#include + +extern int nrn_nthread; +extern int* nrn_prop_dparam_size_; + +namespace neuron::gpu { +namespace { + +constexpr int k_soa_pad = 8; + +int mechanism_padded_count(int count) { + if (count <= 0) { + return 0; + } + return ((count + k_soa_pad - 1) / k_soa_pad) * k_soa_pad; +} + +void record_upload(UploadState& state, + void const* host, + std::size_t count, + std::size_t sizeof_elem) { + state.record(host, count, sizeof_elem); +} + +void upload_net_send_buffer(Memb_list* ml, Memb_list* d_ml, UploadState& state) { + auto* const nsb = ml->_net_send_buffer; + if (!nsb) { + return; + } + auto* const d_nsb = nrn_target_copyin(nsb, 1); + record_upload(state, nsb, 1, sizeof(NetSendBuffer_t)); + nrn_target_memcpy_to_device(&(d_ml->_net_send_buffer), &d_nsb, 1); + + int* d_iptr = nrn_target_copyin(nsb->_sendtype, static_cast(nsb->_size)); + record_upload(state, nsb->_sendtype, static_cast(nsb->_size), sizeof(int)); + nrn_target_memcpy_to_device(&(d_nsb->_sendtype), &d_iptr, 1); + + d_iptr = nrn_target_copyin(nsb->_vdata_index, static_cast(nsb->_size)); + record_upload(state, nsb->_vdata_index, static_cast(nsb->_size), sizeof(int)); + nrn_target_memcpy_to_device(&(d_nsb->_vdata_index), &d_iptr, 1); + + d_iptr = nrn_target_copyin(nsb->_pnt_index, static_cast(nsb->_size)); + record_upload(state, nsb->_pnt_index, static_cast(nsb->_size), sizeof(int)); + nrn_target_memcpy_to_device(&(d_nsb->_pnt_index), &d_iptr, 1); + + d_iptr = nrn_target_copyin(nsb->_weight_index, static_cast(nsb->_size)); + record_upload(state, nsb->_weight_index, static_cast(nsb->_size), sizeof(int)); + nrn_target_memcpy_to_device(&(d_nsb->_weight_index), &d_iptr, 1); + + double* d_dptr = nrn_target_copyin(nsb->_nsb_t, static_cast(nsb->_size)); + record_upload(state, nsb->_nsb_t, static_cast(nsb->_size), sizeof(double)); + nrn_target_memcpy_to_device(&(d_nsb->_nsb_t), &d_dptr, 1); + + d_dptr = nrn_target_copyin(nsb->_nsb_flag, static_cast(nsb->_size)); + record_upload(state, nsb->_nsb_flag, static_cast(nsb->_size), sizeof(double)); + nrn_target_memcpy_to_device(&(d_nsb->_nsb_flag), &d_dptr, 1); +} + +void upload_mechanism_pdata(Memb_list* ml, int type, Memb_list* d_ml, UploadState& state) { + int const szdp = nrn_prop_dparam_size_[type]; + if (!szdp || !ml->pdata || ml->nodecount <= 0) { + return; + } + int const n = ml->nodecount; + int const padded_n = mechanism_padded_count(n); + ml->_nodecount_padded = padded_n; + + std::vector padding_row(static_cast(szdp)); + std::vector device_row_ptrs(static_cast(padded_n), nullptr); + for (int i = 0; i < padded_n; ++i) { + Datum const* host_row = (i < n) ? ml->pdata[i] : padding_row.data(); + Datum* const d_row = nrn_target_copyin(host_row, static_cast(szdp)); + if (i < n) { + record_upload(state, host_row, static_cast(szdp), sizeof(Datum)); + } + device_row_ptrs[static_cast(i)] = d_row; + } + + Datum** const d_pdata_rows = nrn_target_copyin(device_row_ptrs.data(), device_row_ptrs.size()); + record_upload(state, device_row_ptrs.data(), device_row_ptrs.size(), sizeof(Datum*)); + nrn_target_memcpy_to_device(&(d_ml->pdata), &d_pdata_rows, 1); +} + +void upload_mechanism_shell(Memb_list* ml, int type, UploadState& state) { + if (!ml || ml->nodecount <= 0) { + return; + } + + ml->_nodecount_padded = mechanism_padded_count(ml->nodecount); + auto* const d_ml = nrn_target_copyin(ml, 1); + record_upload(state, ml, 1, sizeof(Memb_list)); + + if (ml->nodeindices) { + int* const d_ni = nrn_target_deviceptr(ml->nodeindices); + nrn_target_memcpy_to_device(&(d_ml->nodeindices), &d_ni, 1); + } + + int const thread_size = memb_func[type].thread_size_; + if (thread_size > 0 && ml->_thread) { + Datum* const d_thread = nrn_target_copyin(ml->_thread, + static_cast(thread_size)); + record_upload(state, ml->_thread, static_cast(thread_size), sizeof(Datum)); + nrn_target_memcpy_to_device(&(d_ml->_thread), &d_thread, 1); + } + + upload_mechanism_pdata(ml, type, d_ml, state); + upload_net_send_buffer(ml, d_ml, state); +} + +void upload_thread_ml_list(NrnThread& nt, UploadState& state) { + if (!nt._ml_list) { + return; + } + int const n_type = n_memb_func; + Memb_list** const d_ml_list = nrn_target_copyin(nt._ml_list, static_cast(n_type)); + record_upload(state, nt._ml_list, static_cast(n_type), sizeof(Memb_list*)); + + std::vector device_ptrs(static_cast(n_type), nullptr); + for (int type = 0; type < n_type; ++type) { + if (nt._ml_list[type]) { + device_ptrs[static_cast(type)] = nrn_target_deviceptr(nt._ml_list[type]); + } + } + nrn_target_memcpy_to_device(d_ml_list, device_ptrs.data(), static_cast(n_type)); + + auto* const d_nt = nrn_target_deviceptr(&nt); + nrn_target_memcpy_to_device(&(d_nt->_ml_list), &d_ml_list, 1); +} + +} // namespace + +void upload_mechanism_lists(UploadState& state) { + for (int ith = 0; ith < nrn_nthread; ++ith) { + auto& nt = nrn_threads[ith]; + for (auto* tml = nt.tml; tml; tml = tml->next) { + upload_mechanism_shell(tml->ml, tml->index, state); + } + upload_thread_ml_list(nt, state); + } +} + +} // namespace neuron::gpu \ No newline at end of file diff --git a/src/neuron/model_data.hpp b/src/neuron/model_data.hpp index 4f35a64d5f..ca59aef8fa 100644 --- a/src/neuron/model_data.hpp +++ b/src/neuron/model_data.hpp @@ -5,8 +5,13 @@ #include "neuron/container/node_data.hpp" #include "neuron/model_data_fwd.hpp" +#if defined(NRN_ENABLE_GPU) +#include "neuron/gpu/device_state.hpp" +#endif + #include #include +#include namespace neuron { /** @brief Top-level structure. @@ -161,7 +166,30 @@ struct model_sorted_token { model_sorted_token(cache::Model& cache, container::Node::storage::frozen_token_type node_data_token_) : node_data_token{std::move(node_data_token_)} - , m_cache{cache} {} + , m_cache{cache} { +#if defined(NRN_ENABLE_GPU) + gpu::detail::on_sorted_token_created(); +#endif + } + model_sorted_token(model_sorted_token const& other) + : node_data_token{other.node_data_token} + , mech_data_tokens{other.mech_data_tokens} + , m_cache{other.m_cache} { +#if defined(NRN_ENABLE_GPU) + gpu::detail::on_sorted_token_created(); +#endif + } + model_sorted_token(model_sorted_token&& other) noexcept + : node_data_token{std::move(other.node_data_token)} + , mech_data_tokens{std::move(other.mech_data_tokens)} + , m_cache{other.m_cache} {} + model_sorted_token& operator=(model_sorted_token const&) = delete; + model_sorted_token& operator=(model_sorted_token&&) = delete; + ~model_sorted_token() { +#if defined(NRN_ENABLE_GPU) + gpu::detail::on_sorted_token_destroyed(); +#endif + } [[nodiscard]] cache::Model& cache() { return m_cache; } diff --git a/src/nmodl/codegen/CMakeLists.txt b/src/nmodl/codegen/CMakeLists.txt index bbafaf44dd..12d94ace55 100644 --- a/src/nmodl/codegen/CMakeLists.txt +++ b/src/nmodl/codegen/CMakeLists.txt @@ -7,6 +7,7 @@ add_library( codegen_transform_visitor.cpp codegen_coreneuron_cpp_visitor.cpp codegen_neuron_cpp_visitor.cpp + codegen_neuron_acc_visitor.cpp codegen_cpp_visitor.cpp codegen_compatibility_visitor.cpp codegen_helper_visitor.cpp diff --git a/src/nmodl/codegen/codegen_neuron_acc_visitor.cpp b/src/nmodl/codegen/codegen_neuron_acc_visitor.cpp new file mode 100644 index 0000000000..7163bd1e2d --- /dev/null +++ b/src/nmodl/codegen/codegen_neuron_acc_visitor.cpp @@ -0,0 +1,238 @@ +#include "codegen/codegen_neuron_acc_visitor.hpp" + +#include "ast/block.hpp" +#include "ast/function_call.hpp" +#include "visitors/visitor_utils.hpp" + +namespace nmodl { +namespace codegen { + +std::string CodegenNeuronAccVisitor::backend_name() const { + return "C++-OpenAcc-NEURON"; +} + +void CodegenNeuronAccVisitor::print_standard_includes() { + CodegenNeuronCppVisitor::print_standard_includes(); + printer->add_line("#include "); + printer->add_line("#include "); +} + +void CodegenNeuronAccVisitor::print_parallel_iteration_hint(BlockType type, + const ast::Block* block) { + if (info.artificial_cell) { + return; + } + + // Reuse CPU ivdep path when block uses mutex/protect (atomics + SIMD conflict). + std::vector> nodes; + if (block) { + nodes = collect_nodes(*block, + {ast::AstNodeType::PROTECT_STATEMENT, + ast::AstNodeType::MUTEX_LOCK, + ast::AstNodeType::MUTEX_UNLOCK}); + } + if (!nodes.empty()) { + CodegenNeuronCppVisitor::print_parallel_iteration_hint(type, block); + return; + } + + std::ostringstream present_clause; + present_clause << "present(ml, nt"; + if (type == BlockType::NetReceive) { + present_clause << ", nrb"; + } else { + present_clause << ", nodeindices, thread"; + if (type == BlockType::Equation) { + present_clause << ", vec_rhs, vec_d"; + } + } + present_clause << ')'; + + printer->fmt_line("nrn_pragma_acc(parallel loop {} async(nt->stream_id) if(nt->compute_gpu))", + present_clause.str()); + printer->add_line("nrn_pragma_omp(target teams distribute parallel for if(nt->compute_gpu))"); +} + +void CodegenNeuronAccVisitor::print_kernel_data_present_annotation_block_begin() { + if (!info.artificial_cell) { + printer->add_line("nrn_pragma_acc(data present(nt, ml) if(nt->compute_gpu))"); + printer->add_line("{"); + printer->increase_indent(); + } +} + +void CodegenNeuronAccVisitor::print_kernel_data_present_annotation_block_end() { + if (!info.artificial_cell) { + printer->pop_block(); + } +} + +void CodegenNeuronAccVisitor::print_device_stream_wait() const { + printer->push_block("if(nt->compute_gpu)"); + printer->add_line("nrn_pragma_acc(wait(nt->stream_id))"); + printer->pop_block(); +} + +void CodegenNeuronAccVisitor::print_net_send_buffering_cnt_update() const { + printer->push_block("if (nt->compute_gpu)"); + printer->add_line("nrn_pragma_acc(atomic capture)"); + printer->add_line("nrn_pragma_omp(atomic capture)"); + printer->add_line("i = nsb->_cnt++;"); + printer->chain_block("else"); + printer->add_line("i = nsb->_cnt++;"); + printer->pop_block(); +} + +void CodegenNeuronAccVisitor::print_net_send_buffering_grow() { + printer->add_line("neuron::gpu::net_send_buffer_ensure(ml);"); +} + +void CodegenNeuronAccVisitor::print_net_send_buf_count_update_to_host() const { + printer->add_line("nrn_pragma_acc(update self(nsb->_cnt))"); + printer->add_line("nrn_pragma_omp(target update from(nsb->_cnt))"); +} + +void CodegenNeuronAccVisitor::print_net_send_buf_update_to_host() const { + print_device_stream_wait(); + printer->push_block("if (nsb && nt->compute_gpu)"); + print_net_send_buf_count_update_to_host(); + printer->add_line("neuron::gpu::update_net_send_buffer_on_host(nt, nsb);"); + printer->pop_block(); +} + +void CodegenNeuronAccVisitor::print_net_send_buf_count_update_to_device() const { + printer->push_block("if (nt->compute_gpu)"); + printer->add_line("nrn_pragma_acc(update device(nsb->_cnt))"); + printer->add_line("nrn_pragma_omp(target update to(nsb->_cnt))"); + printer->pop_block(); +} + +void CodegenNeuronAccVisitor::print_net_send_buffering() { + if (!net_send_buffer_required()) { + return; + } + + printer->add_newline(2); + auto args = + "NrnThread* nt, Memb_list* ml, neuron::gpu::NetSendBuffer_t* nsb, int type, " + "intptr_t vdata_ptr, intptr_t weight_ptr, intptr_t point_ptr, double t, double flag"; + printer->fmt_push_block("static inline void net_send_buffering({})", args); + printer->add_line("int i = 0;"); + print_net_send_buffering_grow(); + print_net_send_buffering_cnt_update(); + printer->push_block("if (i < nsb->_size)"); + printer->add_multi_line(R"CODE( + nsb->_sendtype[i] = type; + nsb->_vdata_index[i] = static_cast(vdata_ptr); + nsb->_weight_index[i] = static_cast(weight_ptr); + nsb->_pnt_index[i] = static_cast(point_ptr); + nsb->_nsb_t[i] = t; + nsb->_nsb_flag[i] = flag; + )CODE"); + printer->pop_block(); + printer->pop_block(); +} + +void CodegenNeuronAccVisitor::print_send_event_move() { + printer->add_newline(); + printer->add_line("neuron::gpu::NetSendBuffer_t* nsb = ml->_net_send_buffer;"); + print_net_send_buf_update_to_host(); + printer->add_line("neuron::gpu::deliver_net_send_buffer_events(nt, nsb);"); + print_net_send_buf_count_update_to_device(); +} + +void CodegenNeuronAccVisitor::print_after_nrn_cur_gpu_net_send_flush() { + if (info.net_send_used && !info.artificial_cell) { + print_send_event_move(); + } +} + +void CodegenNeuronAccVisitor::print_compute_functions() { + print_net_send_buffering(); + CodegenNeuronCppVisitor::print_compute_functions(); +} + +void CodegenNeuronAccVisitor::print_net_send_call(const ast::FunctionCall& node) { + auto const& arguments = node.get_arguments(); + const auto& tqitem = get_variable_name("tqitem", /* use_instance */ false); + std::string weight_index = "weight_index"; + std::string point_process = get_variable_name(naming::POINT_PROCESS_VARIABLE, false); + + if (!printing_net_receive && !printing_net_init) { + weight_index = "0"; + if (info.artificial_cell) { + point_process = fmt::format("(Point_process*){}", point_process); + } else { + point_process += ".get()"; + } + } + + if (info.artificial_cell) { + printer->fmt_text("{}(/* tqitem */ &{}, {}, {}, {} + ", + "artcell_net_send", + tqitem, + "nullptr", + point_process, + get_variable_name("t")); + } else { + const auto& t = get_variable_name("t"); + printer->add_text("net_send_buffering("); + std::string weight_ptr = weight_index == "0" ? "0" + : fmt::format("(intptr_t){}", weight_index); + printer->fmt_text( + "nt, ml, ml->_net_send_buffer, 0, (intptr_t)&{}, {}, " + "(intptr_t){}, {}+", + tqitem, + weight_ptr, + point_process, + t); + } + print_vector_elements(arguments, ", "); + printer->add_text(')'); +} + +void CodegenNeuronAccVisitor::print_net_move_call(const ast::FunctionCall& node) { + if (!printing_net_receive && !printing_net_init) { + throw std::runtime_error("Error : net_move only allowed in NET_RECEIVE block"); + } + + const auto& tqitem = get_variable_name("tqitem", false); + const auto& point_process = get_variable_name(naming::POINT_PROCESS_VARIABLE, false); + if (info.artificial_cell) { + printer->fmt_text("artcell_net_move(&{}, {}, ", tqitem, point_process); + print_vector_elements(node.get_arguments(), ", "); + printer->add_text(")"); + return; + } + printer->add_text("net_send_buffering("); + printer->fmt_text( + "nt, ml, ml->_net_send_buffer, 2, (intptr_t)&{}, (intptr_t)-1, " + "(intptr_t){}, ", + tqitem, + point_process); + print_vector_elements(node.get_arguments(), ", "); + printer->add_text(", 0.0, 0.0"); + printer->add_text(")"); +} + +void CodegenNeuronAccVisitor::print_net_event_call(const ast::FunctionCall& node) { + const auto& arguments = node.get_arguments(); + if (info.artificial_cell) { + printer->add_text("net_event(pnt, "); + print_vector_elements(arguments, ", "); + printer->add_text(")"); + return; + } + const auto& point_process = get_variable_name(naming::POINT_PROCESS_VARIABLE, false); + printer->add_text("net_send_buffering("); + printer->fmt_text( + "nt, ml, ml->_net_send_buffer, 1, (intptr_t)-1, (intptr_t)-1, " + "(intptr_t){}, ", + point_process); + print_vector_elements(arguments, ", "); + printer->add_text(", 0.0, 0.0"); + printer->add_text(")"); +} + +} // namespace codegen +} // namespace nmodl \ No newline at end of file diff --git a/src/nmodl/codegen/codegen_neuron_acc_visitor.hpp b/src/nmodl/codegen/codegen_neuron_acc_visitor.hpp new file mode 100644 index 0000000000..f49ecdd20b --- /dev/null +++ b/src/nmodl/codegen/codegen_neuron_acc_visitor.hpp @@ -0,0 +1,60 @@ +#pragma once + +/** + * \file + * \brief \copybrief nmodl::codegen::CodegenNeuronAccVisitor + */ + +#include "codegen/codegen_neuron_cpp_visitor.hpp" + +namespace nmodl { +namespace codegen { + +/** + * \addtogroup codegen_backends + * \{ + */ + +/** + * \class CodegenNeuronAccVisitor + * \brief OpenACC backend for NEURON mechanism codegen (native GPU adoption). + * + * Field mapping follows design §B.4: NEURON sorted SOA pointers and NrnThread::compute_gpu. + */ +class CodegenNeuronAccVisitor: public CodegenNeuronCppVisitor { + public: + using CodegenNeuronCppVisitor::CodegenNeuronCppVisitor; + + protected: + std::string backend_name() const override; + + void print_standard_includes() override; + + void print_parallel_iteration_hint(BlockType type, const ast::Block* block) override; + + void print_kernel_data_present_annotation_block_begin() override; + void print_kernel_data_present_annotation_block_end() override; + + void print_after_nrn_cur_gpu_net_send_flush() override; + + void print_net_send_call(const ast::FunctionCall& node) override; + void print_net_move_call(const ast::FunctionCall& node) override; + void print_net_event_call(const ast::FunctionCall& node) override; + + void print_compute_functions() override; + + private: + void print_device_stream_wait() const; + void print_net_send_buffering(); + void print_send_event_move(); + void print_net_send_buffering_cnt_update() const; + void print_net_send_buffering_grow(); + void print_net_send_buf_count_update_to_host() const; + void print_net_send_buf_update_to_host() const; + void print_net_send_buf_count_update_to_device() const; +}; + +/** \} */ // end of codegen_backends + +} // namespace codegen +} // namespace nmodl diff --git a/src/nmodl/codegen/codegen_neuron_cpp_visitor.cpp b/src/nmodl/codegen/codegen_neuron_cpp_visitor.cpp index 2a799d06a2..ef0b2e673b 100644 --- a/src/nmodl/codegen/codegen_neuron_cpp_visitor.cpp +++ b/src/nmodl/codegen/codegen_neuron_cpp_visitor.cpp @@ -80,6 +80,10 @@ int CodegenNeuronCppVisitor::position_of_int_var(const std::string& name) const /* Backend specific routines */ /****************************************************************************************/ +void CodegenNeuronCppVisitor::print_kernel_data_present_annotation_block_begin() {} + +void CodegenNeuronCppVisitor::print_kernel_data_present_annotation_block_end() {} + bool CodegenNeuronCppVisitor::optimize_ion_variable_copies() const { if (optimize_ionvar_copies) { throw std::runtime_error("Not implemented."); @@ -1672,6 +1676,10 @@ void CodegenNeuronCppVisitor::print_mechanism_register_regular() { printer->add_line("add_nrn_has_net_event(mech_type);"); } + if (info.net_event_used || info.net_send_used) { + printer->add_line("hoc_register_net_send_buffering(mech_type);"); + } + if (info.for_netcon_used) { auto dparam_it = std::find_if(info.semantics.begin(), info.semantics.end(), [](const IndexSemantics& a) { @@ -2031,6 +2039,7 @@ void CodegenNeuronCppVisitor::print_global_function_common_code(BlockType type, {"", "Memb_list*", "", "_ml_arg"}, {"", "int", "", "_type"}}; printer->fmt_push_block("static void {}({})", method, get_parameter_str(args)); + print_kernel_data_present_annotation_block_begin(); print_entrypoint_setup_code_from_memb_list(); printer->add_line("auto nodecount = _ml_arg->nodecount;"); } @@ -2041,6 +2050,7 @@ void CodegenNeuronCppVisitor::print_nrn_init(bool skip_init_check) { print_global_function_common_code(BlockType::Initial); + print_parallel_iteration_hint(BlockType::Initial, info.initial_node); printer->push_block("for (int id = 0; id < nodecount; id++)"); printer->add_line("auto* _ppvar = _ml_arg->pdata[id];"); @@ -2066,6 +2076,7 @@ void CodegenNeuronCppVisitor::print_nrn_init(bool skip_init_check) { } printer->pop_block(); + print_kernel_data_present_annotation_block_end(); printer->pop_block(); } @@ -2291,6 +2302,7 @@ void CodegenNeuronCppVisitor::print_nrn_state() { printer->add_newline(2); print_global_function_common_code(BlockType::State); + print_parallel_iteration_hint(BlockType::State, info.nrn_state_block); printer->push_block("for (int id = 0; id < nodecount; id++)"); printer->add_line("int node_id = node_data.nodeindices[id];"); printer->add_line("auto* _ppvar = _ml_arg->pdata[id];"); @@ -2333,6 +2345,7 @@ void CodegenNeuronCppVisitor::print_nrn_state() { } printer->pop_block(); + print_kernel_data_present_annotation_block_end(); printer->pop_block(); } @@ -2510,7 +2523,7 @@ void CodegenNeuronCppVisitor::print_nrn_cur() { printer->add_newline(2); printer->add_line("/** update current */"); print_global_function_common_code(BlockType::Equation); - // print_channel_iteration_block_parallel_hint(BlockType::Equation, info.breakpoint_node); + print_parallel_iteration_hint(BlockType::Equation, info.breakpoint_node); printer->push_block("for (int id = 0; id < nodecount; id++)"); print_nrn_cur_kernel(*info.breakpoint_node); // print_nrn_cur_matrix_shadow_update(); @@ -2535,10 +2548,13 @@ void CodegenNeuronCppVisitor::print_nrn_cur() { // print_fast_imem_calculation(); // } - // print_kernel_data_present_annotation_block_end(); + print_after_nrn_cur_gpu_net_send_flush(); + print_kernel_data_present_annotation_block_end(); printer->pop_block(); } +void CodegenNeuronCppVisitor::print_after_nrn_cur_gpu_net_send_flush() {} + /****************************************************************************************/ /* Main code printing entry points */ diff --git a/src/nmodl/codegen/codegen_neuron_cpp_visitor.hpp b/src/nmodl/codegen/codegen_neuron_cpp_visitor.hpp index 4e0ebb106b..930eca8c4b 100644 --- a/src/nmodl/codegen/codegen_neuron_cpp_visitor.hpp +++ b/src/nmodl/codegen/codegen_neuron_cpp_visitor.hpp @@ -158,6 +158,20 @@ class CodegenNeuronCppVisitor: public CodegenCppVisitor { */ bool optimize_ion_variable_copies() const override; + /** + * Print accelerator annotations indicating data presence on device. + * Empty for the CPU NEURON backend; overridden by CodegenNeuronAccVisitor. + */ + virtual void print_kernel_data_present_annotation_block_begin(); + + /** + * Print matching block end of accelerator annotations for data presence on device. + */ + virtual void print_kernel_data_present_annotation_block_end(); + + /** Hook after nrn_cur for GPU net_send buffer flush (ACC backend overrides). */ + virtual void print_after_nrn_cur_gpu_net_send_flush(); + /****************************************************************************************/ /* Printing routines for code generation */ /****************************************************************************************/ diff --git a/src/nmodl/main.cpp b/src/nmodl/main.cpp index 7e3e122962..b21e976142 100644 --- a/src/nmodl/main.cpp +++ b/src/nmodl/main.cpp @@ -15,6 +15,7 @@ #include "codegen/codegen_acc_visitor.hpp" #include "codegen/codegen_compatibility_visitor.hpp" #include "codegen/codegen_coreneuron_cpp_visitor.hpp" +#include "codegen/codegen_neuron_acc_visitor.hpp" #include "codegen/codegen_neuron_cpp_visitor.hpp" #include "codegen/codegen_transform_visitor.hpp" #include "config/config.h" @@ -662,6 +663,17 @@ int run_nmodl(int argc, const char* argv[]) { visitor.visit_program(*ast); } + else if (neuron_code && oacc_backend) { + logger->info("Running OpenACC backend code generator for NEURON"); + CodegenNeuronAccVisitor visitor(modfile, + output_stream, + data_type, + optimize_ionvar_copies_codegen, + codegen_cvode, + utils::make_blame(blame_line, blame_level)); + visitor.visit_program(*ast); + } + else if (coreneuron_code && !neuron_code && cpp_backend) { logger->info("Running C++ backend code generator for CoreNEURON"); CodegenCoreneuronCppVisitor visitor(modfile, @@ -672,7 +684,7 @@ int run_nmodl(int argc, const char* argv[]) { visitor.visit_program(*ast); } - else if (neuron_code && cpp_backend) { + else if (neuron_code && cpp_backend && !oacc_backend) { logger->info("Running C++ backend code generator for NEURON"); CodegenNeuronCppVisitor visitor(modfile, output_stream, @@ -686,7 +698,7 @@ int run_nmodl(int argc, const char* argv[]) { else { throw std::runtime_error( "Non valid code generation configuration. Code generation with NMODL is " - "supported for NEURON with C++ backend or CoreNEURON with C++/OpenACC " + "supported for NEURON with C++/OpenACC backends or CoreNEURON with C++/OpenACC " "backends"); } } diff --git a/src/nrniv/CMakeLists.txt b/src/nrniv/CMakeLists.txt index dfc77e9ea3..783b5d8805 100644 --- a/src/nrniv/CMakeLists.txt +++ b/src/nrniv/CMakeLists.txt @@ -397,6 +397,7 @@ include_directories(${NRN_INCLUDE_DIRS}) # All source directories to include # ============================================================================= add_library(nrniv_lib ${NRN_LIBRARY_TYPE} ${NRN_NRNIV_LIB_SRC_FILES}) +target_link_libraries(nrniv_lib PRIVATE neuron_gpu) add_dependencies(nrniv_lib generated_source_files) add_cpp_git_information(nrniv_lib PRIVATE) target_link_libraries(nrniv_lib PRIVATE nrngnu) @@ -464,6 +465,44 @@ endif() set_target_properties(nrniv_lib PROPERTIES EXPORT_NAME nrniv OUTPUT_NAME nrniv) target_compile_features(nrniv_lib PUBLIC cxx_std_17) +# ============================================================================= +# NEURON native GPU: OpenACC compile on libnrniv + CUDA launcher library +# ============================================================================= +if(NRN_ENABLE_GPU AND CORENRN_ENABLE_GPU) + # OpenACC compile flags apply only to cellorder sources: -cuda on all of libnrniv breaks + # InterViews (min/max clash with CUDA headers in external/iv). + separate_arguments(NRN_ACC_FLAGS UNIX_COMMAND "${CORENRN_ACC_COMP_FLAGS}") + set(_nrn_openacc_sources + ${PROJECT_SOURCE_DIR}/src/coreneuron/permute/balance.cpp + ${PROJECT_SOURCE_DIR}/src/coreneuron/permute/cellorder.cpp + ${PROJECT_SOURCE_DIR}/src/coreneuron/permute/cellorder1.cpp + ${PROJECT_SOURCE_DIR}/src/coreneuron/permute/cellorder2.cpp + ${PROJECT_SOURCE_DIR}/src/coreneuron/utils/lpt.cpp + ${PROJECT_SOURCE_DIR}/src/nrnoc/treeset.cpp) + set_source_files_properties( + ${_nrn_openacc_sources} PROPERTIES COMPILE_OPTIONS "${NRN_ACC_FLAGS}" + COMPILE_DEFINITIONS "CORENEURON_ENABLE_GPU;NRN_ENABLE_GPU") + + # OpenACC and explicit CUDA cannot share one .so (NVIDIA forum #210972); reuse coreneuron-cuda + # (cellorder.cu launcher) rather than duplicating the CUDA target in libnrniv. + target_link_libraries(nrniv_lib PRIVATE neuron_gpu_upload) + target_link_libraries(nrniv_lib PUBLIC coreneuron-cuda) + target_link_options(nrniv_lib PUBLIC ${NRN_ACC_FLAGS}) + if(NOT CUDAToolkit_FOUND) + find_package(CUDAToolkit 9.0 QUIET) + endif() + if(CUDAToolkit_FOUND) + set_property( + TARGET nrniv_lib + APPEND + PROPERTY BUILD_RPATH "${CUDAToolkit_LIBRARY_DIR}") + set_property( + TARGET nrniv_lib + APPEND + PROPERTY INSTALL_RPATH "${CUDAToolkit_LIBRARY_DIR}") + endif() +endif() + # ============================================================================= # Link with backward-cpp if enabled # ============================================================================= diff --git a/src/nrnoc/container.cpp b/src/nrnoc/container.cpp index a493a2f5b5..f7cad64e35 100644 --- a/src/nrnoc/container.cpp +++ b/src/nrnoc/container.cpp @@ -4,12 +4,19 @@ #include "neuron/model_data.hpp" #include "section.h" +#if defined(NRN_ENABLE_GPU) +#include "neuron/gpu/device_state.hpp" +#endif + #include #include namespace { void invalidate_cache() { neuron::cache::model.reset(); +#if defined(NRN_ENABLE_GPU) + neuron::gpu::invalidate_device_state(); +#endif } } // namespace namespace neuron { diff --git a/src/nrnoc/fadvance.cpp b/src/nrnoc/fadvance.cpp index c79348f709..4f34f3974b 100644 --- a/src/nrnoc/fadvance.cpp +++ b/src/nrnoc/fadvance.cpp @@ -17,6 +17,14 @@ #include +#if defined(NRN_ENABLE_GPU) +#include "neuron/gpu/config.hpp" +#include "neuron/gpu/device_state.hpp" +#include "neuron/gpu/download.hpp" +#include "neuron/gpu/fadvance_gpu.hpp" +#include "neuron/gpu/net_events.hpp" +#endif + /* after an fadvance from t-dt to t, v is defined at t states that depend on v are defined at t+dt/2 @@ -323,6 +331,12 @@ void nrn_daspk_init_step(double tt, double dteps, int upd) { void nrn_fixed_step(neuron::model_sorted_token const& cache_token) { nrn::Instrumentor::phase p_timestep("timestep"); +#if defined(NRN_ENABLE_GPU) + if (auto const* err = neuron::gpu::native_gpu_configuration_error()) { + hoc_execerror(err, nullptr); + } + neuron::gpu::reset_download_step_counter(); +#endif #if ELIMINATE_T_ROUNDOFF nrn_chk_ndt(); #endif @@ -367,6 +381,9 @@ void nrn_fixed_step(neuron::model_sorted_token const& cache_token) { if (nrn_allthread_handle) { (*nrn_allthread_handle)(); } +#if defined(NRN_ENABLE_GPU) + neuron::gpu::finalize_psolve_download(); +#endif } /* better cache efficiency since a thread can do an entire minimum delay @@ -378,6 +395,9 @@ static int step_group_end; void nrn_fixed_step_group(neuron::model_sorted_token const& cache_token, int n) { int i; +#if defined(NRN_ENABLE_GPU) + neuron::gpu::reset_download_step_counter(); +#endif #if ELIMINATE_T_ROUNDOFF nrn_chk_ndt(); #endif @@ -421,6 +441,9 @@ void nrn_fixed_step_group(neuron::model_sorted_token const& cache_token, int n) while (step_group_end < step_group_n) { /*printf("step_group_end=%d step_group_n=%d\n", step_group_end, step_group_n);*/ nrn_multithread_job(cache_token, nrn_fixed_step_group_thread); +#if defined(NRN_ENABLE_GPU) + neuron::gpu::spike_exchange_after_group(nrn_threads); +#endif if (nrn_allthread_handle) { (*nrn_allthread_handle)(); } @@ -431,6 +454,9 @@ void nrn_fixed_step_group(neuron::model_sorted_token const& cache_token, int n) } } t = nrn_threads[0]._t; +#if defined(NRN_ENABLE_GPU) + neuron::gpu::finalize_psolve_download(); +#endif } static void nrn_fixed_step_group_thread(neuron::model_sorted_token const& cache_token, @@ -455,6 +481,16 @@ static void nrn_fixed_step_group_thread(neuron::model_sorted_token const& cache_ } static void nrn_fixed_step_thread(neuron::model_sorted_token const& cache_token, NrnThread& nt) { +#if defined(NRN_ENABLE_GPU) + // Gap/partrans models use host post-solve (nrnthread_vi_compute_) and partrans + // gather/scatter on host; the full native GPU step is not yet consistent for that + // hybrid. Run the CPU fixed-step body until device gap staging is complete. + if (neuron::gpu::enabled() && neuron::gpu::backend_native() && !nrnthread_v_transfer_) { + neuron::gpu::device_token const& dev = neuron::gpu::ensure_on_device(cache_token); + neuron::gpu::fixed_step_thread(cache_token, dev, nt); + return; + } +#endif auto* const nth = &nt; { nrn::Instrumentor::phase p("deliver-events"); @@ -516,7 +552,14 @@ void nrn_fixed_step_lastpart(neuron::model_sorted_token const& cache_token, NrnT CTADD; { nrn::Instrumentor::phase p("deliver-events"); - nrn_deliver_events(nth); /* up to but not past texit */ +#if defined(NRN_ENABLE_GPU) + if (neuron::gpu::enabled() && neuron::gpu::backend_native()) { + neuron::gpu::deliver_post_step_events_host(nth); + } else +#endif + { + nrn_deliver_events(nth); /* up to but not past texit */ + } } } diff --git a/src/nrnoc/init.cpp b/src/nrnoc/init.cpp index e238d88299..9c4f5df3a5 100644 --- a/src/nrnoc/init.cpp +++ b/src/nrnoc/init.cpp @@ -26,6 +26,10 @@ #include namespace fs = std::filesystem; +#if defined(NRN_ENABLE_GPU) +#include "neuron/gpu/fadvance_gpu.hpp" +#endif + /* change this to correspond to the ../nmodl/nocpout nmodl_version_ string*/ static char nmodl_version_[] = "7.7.0"; @@ -197,6 +201,12 @@ void add_nrn_has_net_event(int mechtype) { nrn_has_net_event_[nrn_has_net_event_cnt_ - 1] = mechtype; } +#if !defined(NRN_ENABLE_GPU) +void hoc_register_net_send_buffering(int type) { + (void) type; +} +#endif + /* values are type numbers of mechanisms which have FOR_NETCONS statement */ int nrn_fornetcon_cnt_; /* how many models have a FOR_NETCONS statement */ int* nrn_fornetcon_type_; /* what are the type numbers */ diff --git a/src/nrnoc/multicore.cpp b/src/nrnoc/multicore.cpp index 10286a6bc0..9e092821e2 100644 --- a/src/nrnoc/multicore.cpp +++ b/src/nrnoc/multicore.cpp @@ -350,6 +350,8 @@ void nrn_threads_create(int n, bool parallel) { nt->_ctime = 0.0; nt->_vcv = 0; nt->_node_data_offset = 0; + nt->compute_gpu = 0; + nt->stream_id = i; } } v_structure_change = 1; diff --git a/src/nrnoc/multicore.h b/src/nrnoc/multicore.h index dec977b9be..0e2b9202dc 100644 --- a/src/nrnoc/multicore.h +++ b/src/nrnoc/multicore.h @@ -71,6 +71,16 @@ struct NrnThread { */ std::size_t _node_data_offset{}; + /** @brief Non-zero when this thread participates in GPU execution (PR 5+). */ + int compute_gpu = 0; + + /** @brief OpenACC/CUDA stream index for async kernels on this thread. */ + int stream_id = 0; + + int _net_send_buffer_size = 0; + int _net_send_buffer_cnt = 0; + int* _net_send_buffer = nullptr; + [[nodiscard]] double* node_a_storage(); [[nodiscard]] double* node_area_storage(); [[nodiscard]] double* node_b_storage(); diff --git a/src/nrnoc/nrniv_mf.h b/src/nrnoc/nrniv_mf.h index 3d28a0cbec..86ddb1d912 100644 --- a/src/nrnoc/nrniv_mf.h +++ b/src/nrnoc/nrniv_mf.h @@ -57,6 +57,7 @@ int point_register_mech(const char**, extern int nrn_get_mechtype(const char*); extern void nrn_writes_conc(int, int); extern void add_nrn_has_net_event(int); +extern void hoc_register_net_send_buffering(int); void hoc_register_cvode(int, nrn_ode_count_t, nrn_ode_map_t, nrn_ode_spec_t, nrn_ode_matsol_t); void hoc_register_synonym(int, nrn_ode_synonym_t); extern void register_destructor(Pvmp); diff --git a/src/nrnoc/nrnoc_ml.h b/src/nrnoc/nrnoc_ml.h index 453bbc8154..b76ea63ddf 100644 --- a/src/nrnoc/nrnoc_ml.h +++ b/src/nrnoc/nrnoc_ml.h @@ -9,6 +9,9 @@ struct Prop; namespace neuron::container { struct generic_data_handle; } +namespace neuron::gpu { +struct NetSendBuffer_t; +} using Datum = neuron::container::generic_data_handle; // Only include a forward declaration to help avoid translated MOD file code relying on its layout @@ -75,7 +78,10 @@ struct Memb_list { Datum** pdata{}; Prop** prop{}; Datum* _thread{}; /* thread specific data (when static is no good) */ + neuron::gpu::NetSendBuffer_t* _net_send_buffer = nullptr; int nodecount{}; + /** Padded instance count for GPU SoA kernels (set during device upload). */ + int _nodecount_padded{}; /** * @brief Get a vector of double* representing the model data. * diff --git a/src/nrnoc/treeset.cpp b/src/nrnoc/treeset.cpp index e24a45b196..52e8d06acc 100644 --- a/src/nrnoc/treeset.cpp +++ b/src/nrnoc/treeset.cpp @@ -22,6 +22,11 @@ #include "utils/profile/profiler_interface.h" #include "multicore.h" +#include "neuron/gpu/offload.hpp" +#if defined(NRN_ENABLE_GPU) +#include "neuron/gpu/sync.hpp" +#endif + #include #include #include @@ -401,6 +406,10 @@ void nrn_rhs(neuron::model_sorted_token const& cache_token, NrnThread& nt) { for (i = i1; i < i3; ++i) { NODERHS(_nt->_v_node[i]) = 0.; } + } else if (_nt->compute_gpu && i3 > i1) { + for (i = i1; i < i3; ++i) { + vec_rhs[i] = 0.; + } } else { for (i = i1; i < i3; ++i) { vec_rhs[i] = 0.; @@ -408,8 +417,14 @@ void nrn_rhs(neuron::model_sorted_token const& cache_token, NrnThread& nt) { } auto const vec_sav_rhs = _nt->node_sav_rhs_storage(); if (vec_sav_rhs) { - for (i = i1; i < i3; ++i) { - vec_sav_rhs[i] = 0.; + if (_nt->compute_gpu && i3 > i1) { + for (i = i1; i < i3; ++i) { + vec_sav_rhs[i] = 0.; + } + } else { + for (i = i1; i < i3; ++i) { + vec_sav_rhs[i] = 0.; + } } } @@ -436,12 +451,22 @@ void nrn_rhs(neuron::model_sorted_token const& cache_token, NrnThread& nt) { } activsynapse_rhs(); +#if defined(NRN_ENABLE_GPU) + neuron::gpu::sync_matrix_to_device_after_mechanisms(nt); +#endif + if (vec_sav_rhs) { /* vec_sav_rhs has only the contribution of electrode current here we transform so it only has membrane current contribution */ - for (i = i1; i < i3; ++i) { - vec_sav_rhs[i] -= vec_rhs[i]; + if (_nt->compute_gpu && i3 > i1) { + for (i = i1; i < i3; ++i) { + vec_sav_rhs[i] -= vec_rhs[i]; + } + } else { + for (i = i1; i < i3; ++i) { + vec_sav_rhs[i] -= vec_rhs[i]; + } } } #if EXTRACELLULAR @@ -460,6 +485,12 @@ void nrn_rhs(neuron::model_sorted_token const& cache_token, NrnThread& nt) { activstim_rhs(); activclamp_rhs(); +#if defined(NRN_ENABLE_GPU) + neuron::gpu::sync_matrix_to_device_after_mechanisms(nt); + if (_nt->compute_gpu && i3 > i2) { + neuron::gpu::sync_voltages_to_device_before_axial(nt); + } +#endif /* now the internal axial currents. The extracellular mechanism contribution is already done. rhs += ai_j*(vi_j - vi) @@ -468,13 +499,29 @@ void nrn_rhs(neuron::model_sorted_token const& cache_token, NrnThread& nt) { auto* const vec_b = nt.node_b_storage(); auto* const vec_v = nt.node_voltage_storage(); auto* const parent_i = nt._v_parent_index; + nrn_pragma_acc(parallel loop present(vec_rhs [0:i3], + vec_a [0:i3], + vec_b [0:i3], + vec_v [0:i3], + parent_i [0:i3]) if (_nt->compute_gpu && i3 > i2) + async(_nt->stream_id)) + nrn_pragma_omp(target teams distribute parallel for if(_nt->compute_gpu)) for (i = i2; i < i3; ++i) { auto const pi = parent_i[i]; auto const dv = vec_v[pi] - vec_v[i]; // our connection coefficients are negative so + nrn_pragma_acc(atomic update) + nrn_pragma_omp(atomic update) vec_rhs[i] -= vec_b[i] * dv; + nrn_pragma_acc(atomic update) + nrn_pragma_omp(atomic update) vec_rhs[pi] += vec_a[i] * dv; } +#if defined(NRN_ENABLE_GPU) + if (_nt->compute_gpu && i3 > i2) { + nrn_pragma_acc(wait(_nt->stream_id)) + } +#endif } /* calculate left hand side of @@ -505,14 +552,26 @@ void nrn_lhs(neuron::model_sorted_token const& sorted_token, NrnThread& nt) { // Make sure the SoA node diagonals are also zeroed (is this needed?) auto* const vec_d = _nt->node_d_storage(); - for (int i = i1; i < i3; ++i) { - vec_d[i] = 0.; + if (_nt->compute_gpu && i3 > i1) { + for (int i = i1; i < i3; ++i) { + vec_d[i] = 0.; + } + } else { + for (int i = i1; i < i3; ++i) { + vec_d[i] = 0.; + } } auto const vec_sav_d = _nt->node_sav_d_storage(); if (vec_sav_d) { - for (i = i1; i < i3; ++i) { - vec_sav_d[i] = 0.; + if (_nt->compute_gpu && i3 > i1) { + for (i = i1; i < i3; ++i) { + vec_sav_d[i] = 0.; + } + } else { + for (i = i1; i < i3; ++i) { + vec_sav_d[i] = 0.; + } } } @@ -539,14 +598,24 @@ void nrn_lhs(neuron::model_sorted_token const& sorted_token, NrnThread& nt) { nrn_cap_jacob(sorted_token, _nt, _nt->tml->ml); } +#if defined(NRN_ENABLE_GPU) + neuron::gpu::sync_diagonal_to_device_after_mechanisms(nt); +#endif + activsynapse_lhs(); if (vec_sav_d) { /* vec_sav_d has only the contribution of electrode current here we transform so it only has membrane current contribution */ - for (i = i1; i < i3; ++i) { - vec_sav_d[i] = vec_d[i] - vec_sav_d[i]; + if (_nt->compute_gpu && i3 > i1) { + for (i = i1; i < i3; ++i) { + vec_sav_d[i] = vec_d[i] - vec_sav_d[i]; + } + } else { + for (i = i1; i < i3; ++i) { + vec_sav_d[i] = vec_d[i] - vec_sav_d[i]; + } } } #if EXTRACELLULAR @@ -565,6 +634,9 @@ void nrn_lhs(neuron::model_sorted_token const& sorted_token, NrnThread& nt) { /* at this point d contains all the membrane conductances */ +#if defined(NRN_ENABLE_GPU) + neuron::gpu::sync_diagonal_to_device_after_mechanisms(nt); +#endif /* now add the axial currents */ if (use_sparse13) { @@ -588,10 +660,26 @@ void nrn_lhs(neuron::model_sorted_token const& sorted_token, NrnThread& nt) { } else { auto* const vec_a = _nt->node_a_storage(); auto* const vec_b = _nt->node_b_storage(); + auto* const parent_i = _nt->_v_parent_index; + nrn_pragma_acc(parallel loop present(vec_d [0:i3], + vec_a [0:i3], + vec_b [0:i3], + parent_i [0:i3]) if (_nt->compute_gpu && i3 > i2) + async(_nt->stream_id)) + nrn_pragma_omp(target teams distribute parallel for if(_nt->compute_gpu)) for (i = i2; i < i3; ++i) { + nrn_pragma_acc(atomic update) + nrn_pragma_omp(atomic update) vec_d[i] -= vec_b[i]; - vec_d[_nt->_v_parent_index[i]] -= vec_a[i]; + nrn_pragma_acc(atomic update) + nrn_pragma_omp(atomic update) + vec_d[parent_i[i]] -= vec_a[i]; + } +#if defined(NRN_ENABLE_GPU) + if (_nt->compute_gpu && i3 > i2) { + nrn_pragma_acc(wait(_nt->stream_id)) } +#endif } } @@ -600,6 +688,9 @@ void setup_tree_matrix(neuron::model_sorted_token const& cache_token, NrnThread& nrn::Instrumentor::phase _{"setup-tree-matrix"}; nrn_rhs(cache_token, nt); nrn_lhs(cache_token, nt); +#if defined(NRN_ENABLE_GPU) + neuron::gpu::sync_matrix_to_host_before_solve(nt); +#endif nrn_nonvint_block_current(nt.end, nt.node_rhs_storage(), nt.id); nrn_nonvint_block_conductance(nt.end, nt.node_d_storage(), nt.id); } diff --git a/src/oc/mech_api.h b/src/oc/mech_api.h index 8d3ed11ecd..dcaae211e7 100644 --- a/src/oc/mech_api.h +++ b/src/oc/mech_api.h @@ -20,3 +20,5 @@ #include // nocmodl uses std::isnan #include // nocmodl uses std::cerr + +void hoc_register_net_send_buffering(int); diff --git a/src/parallel/ocbbs.cpp b/src/parallel/ocbbs.cpp index 845bf2144a..70847eb557 100644 --- a/src/parallel/ocbbs.cpp +++ b/src/parallel/ocbbs.cpp @@ -16,6 +16,12 @@ #include #include +#if defined(NRN_ENABLE_GPU) +#include "neuron/gpu/config.hpp" +#include "neuron/gpu/device_assign.hpp" +#include "neuron/gpu/download.hpp" +#endif + #undef MD #define MD 2147483647. @@ -933,6 +939,61 @@ static double optimize_node_order(void*) { return double(neuron::interleave_permute_type); } +#if defined(NRN_ENABLE_GPU) + +static double gpu_enable(void*) { + hoc_return_type_code = HocReturnType::boolean; + if (ifarg(1)) { + neuron::gpu::set_enable(chkarg(1, 0, 1) != 0.0); + if (auto const* err = neuron::gpu::native_gpu_configuration_error()) { + hoc_execerror(err, nullptr); + } + } + return double(neuron::gpu::enabled()); +} + +static const char** gpu_backend(void*) { + if (ifarg(1)) { + neuron::gpu::set_backend(hoc_gargstr(1)); + if (auto const* err = neuron::gpu::native_gpu_configuration_error()) { + hoc_execerror(err, nullptr); + } + } + static char native_storage[] = "native"; + static char coreneuron_storage[] = "coreneuron"; + static char* backend_ptr = coreneuron_storage; + backend_ptr = neuron::gpu::backend_native() ? native_storage : coreneuron_storage; + return (const char**) &backend_ptr; +} + +static double gpu_device_count(void*) { + hoc_return_type_code = HocReturnType::integer; + if (ifarg(1)) { + neuron::gpu::set_device_count(static_cast(chkarg(1, 0, 1024))); + } + return double(neuron::gpu::device_count()); +} + +static double gpu_assign_device(void*) { + neuron::gpu::assign_device(); + return 1.0; +} + +static double gpu_device_id(void*) { + hoc_return_type_code = HocReturnType::integer; + return double(neuron::gpu::assigned_device_id()); +} + +static double gpu_download_flush_interval(void*) { + hoc_return_type_code = HocReturnType::integer; + if (ifarg(1)) { + neuron::gpu::set_download_flush_interval(static_cast(chkarg(1, 0, 1e9))); + } + return double(neuron::gpu::download_flush_interval()); +} + +#endif + static double sec_in_thread(void*) { hoc_return_type_code = HocReturnType::boolean; Section* sec = chk_access(); @@ -1022,88 +1083,101 @@ static Object** gid_connect(void* v) { return bbs->gid_connect(int(chkarg(1, 0, MD))); } -static Member_func members[] = {{"submit", submit}, - {"working", working}, - {"retval", retval}, - {"userid", userid}, - {"pack", pack}, - {"post", post}, - {"unpack", unpack}, - {"upkscalar", upkscalar}, - {"take", take}, - {"look", look}, - {"look_take", look_take}, - {"runworker", worker}, - {"master_works_on_jobs", master_works}, - {"done", done}, - {"id", nrn_rank}, - {"nhost", nhost}, - {"id_world", rank_world}, - {"nhost_world", nhost_world}, - {"id_bbs", rank_bbs}, - {"nhost_bbs", nhost_bbs}, - {"subworlds", subworlds}, - {"context", context}, - - {"time", pctime}, - {"wait_time", wait_time}, - {"step_time", step_time}, - {"step_wait", step_wait}, - {"send_time", send_time}, - {"event_time", event_time}, - {"integ_time", integ_time}, - {"vtransfer_time", vtransfer_time}, - {"mech_time", mech_time}, - {"timeout", set_timeout}, - {"mpiabort_on_error", set_mpiabort_on_error}, - - {"set_gid2node", set_gid2node}, - {"gid_exists", gid_exists}, - {"outputcell", outputcell}, - {"cell", cell}, - {"threshold", threshold}, - {"spike_record", spike_record}, - {"psolve", psolve}, - {"set_maxstep", set_maxstep}, - {"spike_statistics", spike_stat}, - {"max_histogram", maxhist}, - {"spike_compress", spcompress}, - {"gid_clear", gid_clear}, - {"prcellstate", prcellstate}, - - {"source_var", source_var}, - {"target_var", target_var}, - {"setup_transfer", setup_transfer}, - {"splitcell_connect", splitcell_connect}, - {"multisplit", multisplit}, - - {"barrier", barrier}, - {"allreduce", allreduce}, - {"allgather", allgather}, - {"alltoall", alltoall}, - {"broadcast", broadcast}, - - {"nthread", nthrd}, - {"nworker", number_of_worker_threads}, - {"partition", partition}, - {"thread_stat", thread_stat}, - {"thread_busywait", thread_busywait}, - {"thread_how_many_proc", thread_how_many_proc}, - {"optimize_node_order", optimize_node_order}, - {"sec_in_thread", sec_in_thread}, - {"thread_ctime", thread_ctime}, - {"dt", thread_dt}, - {"t", nrn_thread_t}, - - {"nrnbbcore_write", nrncorewrite_argvec}, - {"nrncore_write", nrncorewrite_argappend}, - {"nrnbbcore_register_mapping", nrnbbcore_register_mapping}, - {"nrncore_run", nrncorerun}, - {"print_memory_stats", print_memory_stats}, - - {0, 0}}; - -static Member_ret_str_func retstr_members[] = {{"upkstr", upkstr}, {0, 0}}; +static Member_func members[] = +{ {"submit", submit}, + {"working", working}, + {"retval", retval}, + {"userid", userid}, + {"pack", pack}, + {"post", post}, + {"unpack", unpack}, + {"upkscalar", upkscalar}, + {"take", take}, + {"look", look}, + {"look_take", look_take}, + {"runworker", worker}, + {"master_works_on_jobs", master_works}, + {"done", done}, + {"id", nrn_rank}, + {"nhost", nhost}, + {"id_world", rank_world}, + {"nhost_world", nhost_world}, + {"id_bbs", rank_bbs}, + {"nhost_bbs", nhost_bbs}, + {"subworlds", subworlds}, + {"context", context}, + + {"time", pctime}, + {"wait_time", wait_time}, + {"step_time", step_time}, + {"step_wait", step_wait}, + {"send_time", send_time}, + {"event_time", event_time}, + {"integ_time", integ_time}, + {"vtransfer_time", vtransfer_time}, + {"mech_time", mech_time}, + {"timeout", set_timeout}, + {"mpiabort_on_error", set_mpiabort_on_error}, + + {"set_gid2node", set_gid2node}, + {"gid_exists", gid_exists}, + {"outputcell", outputcell}, + {"cell", cell}, + {"threshold", threshold}, + {"spike_record", spike_record}, + {"psolve", psolve}, + {"set_maxstep", set_maxstep}, + {"spike_statistics", spike_stat}, + {"max_histogram", maxhist}, + {"spike_compress", spcompress}, + {"gid_clear", gid_clear}, + {"prcellstate", prcellstate}, + + {"source_var", source_var}, + {"target_var", target_var}, + {"setup_transfer", setup_transfer}, + {"splitcell_connect", splitcell_connect}, + {"multisplit", multisplit}, + + {"barrier", barrier}, + {"allreduce", allreduce}, + {"allgather", allgather}, + {"alltoall", alltoall}, + {"broadcast", broadcast}, + + {"nthread", nthrd}, + {"nworker", number_of_worker_threads}, + {"partition", partition}, + {"thread_stat", thread_stat}, + {"thread_busywait", thread_busywait}, + {"thread_how_many_proc", thread_how_many_proc}, + {"optimize_node_order", optimize_node_order}, +#if defined(NRN_ENABLE_GPU) + {"gpu_enable", gpu_enable}, + {"gpu_device_count", gpu_device_count}, + {"gpu_assign_device", gpu_assign_device}, + {"gpu_device_id", gpu_device_id}, + {"gpu_download_flush_interval", gpu_download_flush_interval}, +#endif + {"sec_in_thread", sec_in_thread}, + {"thread_ctime", thread_ctime}, + {"dt", thread_dt}, + {"t", nrn_thread_t}, + + {"nrnbbcore_write", nrncorewrite_argvec}, + {"nrncore_write", nrncorewrite_argappend}, + {"nrnbbcore_register_mapping", nrnbbcore_register_mapping}, + {"nrncore_run", nrncorerun}, + {"print_memory_stats", print_memory_stats}, + + {0, 0} }; + +static Member_ret_str_func retstr_members[] = +{ {"upkstr", upkstr}, +#if defined(NRN_ENABLE_GPU) + {"gpu_backend", gpu_backend}, +#endif + {0, 0} }; static Member_ret_obj_func retobj_members[] = {{"upkvec", upkvec}, {"gid2obj", gid2obj}, diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index ffa28da9b2..5b4b020e4c 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -81,6 +81,148 @@ if(TARGET nrn-benchmarks AND NRN_ENABLE_PERFORMANCE_TESTS) COMMAND $ --processors=16) endif() +if(NRN_ENABLE_GPU AND CORENRN_ENABLE_GPU) + separate_arguments(_test_gpu_offload_flags UNIX_COMMAND "${CORENRN_ACC_COMP_FLAGS}") + add_executable(testneuron_gpu_offload unit_tests/gpu/offload_main.cpp unit_tests/gpu/offload.cpp) + cpp_cc_configure_sanitizers(TARGET testneuron_gpu_offload) + target_link_libraries(testneuron_gpu_offload PRIVATE Catch2::Catch2 neuron_gpu) + target_compile_definitions(testneuron_gpu_offload PRIVATE NRN_ENABLE_GPU) + target_compile_options(testneuron_gpu_offload PRIVATE ${_test_gpu_offload_flags}) + target_link_options(testneuron_gpu_offload PUBLIC ${_test_gpu_offload_flags}) + if(CUDAToolkit_FOUND) + set_property( + TARGET testneuron_gpu_offload + APPEND + PROPERTY BUILD_RPATH "${CUDAToolkit_LIBRARY_DIR}") + endif() + nrn_add_test( + GROUP unit_tests + NAME gpu_offload + REQUIRES gpu + COMMAND $ "[gpu][offload]") + + add_executable( + testneuron_gpu_device_state unit_tests/gpu/offload_main.cpp unit_tests/gpu/device_state.cpp + unit_tests/gpu/upload_link_stubs.cpp) + cpp_cc_configure_sanitizers(TARGET testneuron_gpu_device_state) + target_include_directories( + testneuron_gpu_device_state + PRIVATE ${PROJECT_SOURCE_DIR}/external/fmt/include ${PROJECT_SOURCE_DIR}/external/backward + ${CMAKE_BINARY_DIR}/src/nrnoc) + target_link_libraries(testneuron_gpu_device_state PRIVATE Catch2::Catch2 neuron_gpu + neuron_gpu_upload) + target_compile_definitions(testneuron_gpu_device_state PRIVATE NRN_ENABLE_GPU) + target_compile_options(testneuron_gpu_device_state PRIVATE ${_test_gpu_offload_flags}) + target_link_options(testneuron_gpu_device_state PUBLIC ${_test_gpu_offload_flags}) + if(CUDAToolkit_FOUND) + set_property( + TARGET testneuron_gpu_device_state + APPEND + PROPERTY BUILD_RPATH "${CUDAToolkit_LIBRARY_DIR}") + endif() + nrn_add_test( + GROUP unit_tests + NAME gpu_device_state + REQUIRES gpu + COMMAND $ "[gpu][device_state]") + + add_executable(testneuron_gpu_config unit_tests/gpu/offload_main.cpp unit_tests/gpu/config.cpp) + cpp_cc_configure_sanitizers(TARGET testneuron_gpu_config) + target_link_libraries(testneuron_gpu_config PRIVATE Catch2::Catch2 neuron_gpu) + target_compile_definitions(testneuron_gpu_config PRIVATE NRN_ENABLE_GPU) + nrn_add_test( + GROUP unit_tests + NAME gpu_config + REQUIRES gpu + COMMAND $ "[gpu][config]") + + add_executable(testneuron_gpu_device_assign unit_tests/gpu/offload_main.cpp + unit_tests/gpu/device_assign.cpp) + cpp_cc_configure_sanitizers(TARGET testneuron_gpu_device_assign) + target_link_libraries(testneuron_gpu_device_assign PRIVATE Catch2::Catch2 neuron_gpu) + target_compile_definitions(testneuron_gpu_device_assign PRIVATE NRN_ENABLE_GPU) + target_compile_options(testneuron_gpu_device_assign PRIVATE ${_test_gpu_offload_flags}) + target_link_options(testneuron_gpu_device_assign PUBLIC ${_test_gpu_offload_flags}) + nrn_add_test( + GROUP unit_tests + NAME gpu_device_assign + REQUIRES gpu + COMMAND $ "[gpu][device_assign]") + + add_executable( + testneuron_gpu_upload + unit_tests/gpu/offload_main.cpp unit_tests/gpu/upload.cpp unit_tests/gpu/upload_link_stubs.cpp + unit_tests/gpu/interleave_info_stub.cpp) + cpp_cc_configure_sanitizers(TARGET testneuron_gpu_upload) + target_include_directories( + testneuron_gpu_upload PRIVATE ${PROJECT_SOURCE_DIR}/external/fmt/include + ${CMAKE_BINARY_DIR}/src/nrnoc) + target_link_libraries(testneuron_gpu_upload PRIVATE Catch2::Catch2 neuron_gpu neuron_gpu_upload + fmt::fmt) + target_compile_definitions(testneuron_gpu_upload PRIVATE NRN_ENABLE_GPU) + target_compile_options(testneuron_gpu_upload PRIVATE ${_test_gpu_offload_flags}) + target_link_options(testneuron_gpu_upload PUBLIC ${_test_gpu_offload_flags}) + nrn_add_test( + GROUP unit_tests + NAME gpu_upload + REQUIRES gpu + COMMAND $ "[gpu][upload]") + + add_executable( + testneuron_gpu_fadvance + unit_tests/gpu/offload_main.cpp unit_tests/gpu/fadvance.cpp + unit_tests/gpu/upload_link_stubs.cpp unit_tests/gpu/fadvance_link_stubs.cpp) + cpp_cc_configure_sanitizers(TARGET testneuron_gpu_fadvance) + target_include_directories( + testneuron_gpu_fadvance PRIVATE ${PROJECT_SOURCE_DIR}/external/fmt/include + ${CMAKE_BINARY_DIR}/src/nrnoc) + target_link_libraries(testneuron_gpu_fadvance PRIVATE Catch2::Catch2 neuron_gpu neuron_gpu_upload + fmt::fmt) + target_compile_definitions(testneuron_gpu_fadvance PRIVATE NRN_ENABLE_GPU) + target_compile_options(testneuron_gpu_fadvance PRIVATE ${_test_gpu_offload_flags}) + target_link_options(testneuron_gpu_fadvance PUBLIC ${_test_gpu_offload_flags}) + nrn_add_test( + GROUP unit_tests + NAME gpu_fadvance + REQUIRES gpu + COMMAND $ "[gpu][fadvance]") + + add_executable(testneuron_gpu_net_events unit_tests/gpu/offload_main.cpp + unit_tests/gpu/net_events.cpp) + cpp_cc_configure_sanitizers(TARGET testneuron_gpu_net_events) + target_include_directories( + testneuron_gpu_net_events PRIVATE ${PROJECT_SOURCE_DIR}/external/fmt/include + ${CMAKE_BINARY_DIR}/src/nrnoc) + target_link_libraries(testneuron_gpu_net_events PRIVATE Catch2::Catch2 neuron_gpu + neuron_gpu_upload fmt::fmt) + target_compile_definitions(testneuron_gpu_net_events PRIVATE NRN_ENABLE_GPU) + target_compile_options(testneuron_gpu_net_events PRIVATE ${_test_gpu_offload_flags}) + target_link_options(testneuron_gpu_net_events PUBLIC ${_test_gpu_offload_flags}) + nrn_add_test( + GROUP unit_tests + NAME gpu_net_events + REQUIRES gpu + COMMAND $ "[gpu][net_events]") +endif() + +# Compile-only guard: cellorder.cpp OpenACC + NRN_ENABLE_GPU (solve_interleaved GPU dispatch). +if(NRN_ENABLE_GPU AND CORENRN_ENABLE_GPU) + add_library(cellorder_openacc_smoke OBJECT + ${PROJECT_SOURCE_DIR}/src/coreneuron/permute/cellorder.cpp) + separate_arguments(_cellorder_openacc_flags UNIX_COMMAND "${CORENRN_ACC_COMP_FLAGS}") + target_compile_options(cellorder_openacc_smoke PRIVATE ${_cellorder_openacc_flags}) + target_compile_definitions(cellorder_openacc_smoke PRIVATE CORENEURON_ENABLE_GPU NRN_ENABLE_GPU) + target_include_directories(cellorder_openacc_smoke PRIVATE ${NRN_INCLUDE_DIRS} + ${CMAKE_BINARY_DIR}/generated) + target_include_directories(cellorder_openacc_smoke SYSTEM + PRIVATE ${PROJECT_SOURCE_DIR}/external/fmt/include) + nrn_add_test( + GROUP unit_tests + NAME cellorder_openacc_smoke + REQUIRES gpu + COMMAND ${CMAKE_COMMAND} --build ${CMAKE_BINARY_DIR} --target cellorder_openacc_smoke) +endif() + # ============================================================================= # Add ringtest # ============================================================================= @@ -243,7 +385,8 @@ if(NRN_ENABLE_PYTHON) PRELOAD_SANITIZER ENVIRONMENT "CC=${CMAKE_C_COMPILER}" COMMAND "${exe}" ${pytest} "./test/${group}" - SCRIPT_PATTERNS "test/${group}/*.json" "test/${group}/*.py") + SCRIPT_PATTERNS "test/${group}/*.json" "test/${group}/*.py" + "test/coreneuron/backend_helper.py") endforeach() endforeach() @@ -457,6 +600,8 @@ if(NRN_ENABLE_PYTHON) ${MPIEXEC_POSTFLAGS}) endif() + set(coreneuron_backend_helper_script test/coreneuron/backend_helper.py) + # This test uses the standard NEURON installation (without nrnivmodl having been run) if(CORENRN_ENABLE_SHARED) nrn_add_test_group( @@ -468,7 +613,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_standalone NAME test_nrn_corenrn_standalone REQUIRES coreneuron ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_psolve.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_psolve.py ENVIRONMENT COVERAGE_FILE=.coverage.coreneuron_standalone_test_psolve_py ${sonata_zero_gid_env} ${nrnpython_mpi_env} COMMAND ${modtests_launch_py} test/coreneuron/test_psolve.py) @@ -517,7 +662,7 @@ if(NRN_ENABLE_PYTHON) CORENEURON NAME coreneuron_modtests # This get used in 4 tests so make it the default and override in other tests. - SCRIPT_PATTERNS test/coreneuron/test_spikes.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_spikes.py MODFILE_PATTERNS "test/coreneuron/mod files/*.mod" "test/coreneuron/mod files/axial.inc" test/pytest_coreneuron/unitstest.mod test/pytest_coreneuron/version_macros.mod @@ -546,7 +691,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME fornetcon_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_fornetcon.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_fornetcon.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_fornetcon_py COMMAND ${modtests_launch_py} test/coreneuron/test_fornetcon.py) @@ -554,7 +699,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME direct_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_direct.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_direct.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_direct_py COMMAND ${modtests_launch_py} test/coreneuron/test_direct.py) @@ -584,7 +729,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME fast_imem_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/pytest_coreneuron/test_fast_imem.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/pytest_coreneuron/test_fast_imem.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_fast_imem_py COMMAND ${modtests_launch_py} test/pytest_coreneuron/test_fast_imem.py) @@ -592,7 +737,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME datareturn_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_datareturn.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_datareturn.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_datareturn_py COMMAND ${modtests_launch_py} test/coreneuron/test_datareturn.py) @@ -600,7 +745,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME test_units_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_units.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_units.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_test_units_py COMMAND ${modtests_launch_py} test/coreneuron/test_units.py) @@ -608,7 +753,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME test_netmove_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_netmove.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_netmove.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_test_netmove_py COMMAND ${modtests_launch_py} test/coreneuron/test_netmove.py) @@ -616,7 +761,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME test_pointer_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_pointer.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_pointer.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_test_pointer_py COMMAND ${modtests_launch_py} test/coreneuron/test_pointer.py) @@ -624,7 +769,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME test_watchrange_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_watchrange.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_watchrange.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_test_watchrange_py COMMAND ${modtests_launch_py} test/coreneuron/test_watchrange.py) @@ -632,7 +777,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME test_psolve_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_psolve.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_psolve.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_test_psolve_py COMMAND ${modtests_launch_py} test/coreneuron/test_psolve.py) @@ -640,7 +785,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME test_ba_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_ba.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_ba.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_test_ba_py COMMAND ${modtests_launch_py} test/coreneuron/test_ba.py) @@ -648,7 +793,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME test_nmodlrandom_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_nmodlrandom.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/coreneuron/test_nmodlrandom.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_test_nmodlrandom_py COMMAND ${modtests_launch_py} test/coreneuron/test_nmodlrandom.py) @@ -666,7 +811,7 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME test_natrans_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/gjtests/test_natrans.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} test/gjtests/test_natrans.py ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env} COVERAGE_FILE=.coverage.coreneuron_test_natrans_py COMMAND ${modtests_launch_py} test/gjtests/test_natrans.py) @@ -676,7 +821,8 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME array_variable_transfer_run_mode_${run_mode}_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_array_variables_transfer.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} + test/coreneuron/test_array_variables_transfer.py ENVIRONMENT ${processor_env} NRN_TEST_RUN_MODE=${run_mode} NRN_TEST_FILE_MODE=false COMMAND ${modtests_launch_py} test/coreneuron/test_array_variables_transfer.py) endforeach() @@ -685,7 +831,8 @@ if(NRN_ENABLE_PYTHON) GROUP coreneuron_modtests NAME array_variable_transfer_file_mode_py_${processor} REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer} - SCRIPT_PATTERNS test/coreneuron/test_array_variables_transfer.py + SCRIPT_PATTERNS ${coreneuron_backend_helper_script} + test/coreneuron/test_array_variables_transfer.py ENVIRONMENT ${processor_env} NRN_TEST_RUN_MODE=0 NRN_TEST_FILE_MODE=true COMMAND ${modtests_launch_py} test/coreneuron/test_array_variables_transfer.py) @@ -764,6 +911,76 @@ if(NRN_ENABLE_PYTHON) COMMAND ${modtests_launch_py} test/nmodl/test_kinetic.py) endforeach() + # G4 native GPU modtests (NRN_GPU_BACKEND_TEST=native): parity with coreneuron_modtests *_py_gpu. + if(NRN_ENABLE_GPU) + set(native_gpu_test_env NRN_GPU_BACKEND_TEST=native NRN_GPU_PERMUTE=2 OMP_NUM_THREADS=1) + set(g4_native_helper_script ${coreneuron_backend_helper_script}) + # Native GPU exercises OpenACC in libnrniv; launch via special (not bare python/pytest). + set(native_modtests_launch_py special -notatty -python) + nrn_add_test( + GROUP unit_tests + NAME gpu_device_assign_mpi + REQUIRES gpu mpi python + PROCESSORS 2 + ENVIRONMENT ${native_gpu_test_env} NEURON_INIT_MPI=1 + SCRIPT_PATTERNS test/pyscripts/test_gpu_device_assign_mpi.py + COMMAND + ${MPIEXEC_NAME} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_OVERSUBSCRIBE} ${MPIEXEC_PREFLAGS} + nrniv ${MPIEXEC_POSTFLAGS} -mpi -notatty -python + test/pyscripts/test_gpu_device_assign_mpi.py) + foreach( + native_gpu_test + fornetcon:test/coreneuron/test_fornetcon.py + direct:test/coreneuron/test_direct.py + spikes:test/coreneuron/test_spikes.py + fast_imem:test/pytest_coreneuron/test_fast_imem.py + datareturn:test/coreneuron/test_datareturn.py + test_units:test/coreneuron/test_units.py + test_netmove:test/coreneuron/test_netmove.py + test_pointer:test/coreneuron/test_pointer.py + test_watchrange:test/coreneuron/test_watchrange.py + test_psolve:test/coreneuron/test_psolve.py + test_ba:test/coreneuron/test_ba.py + test_nmodlrandom:test/coreneuron/test_nmodlrandom.py + test_nmodlrandom_syntax:test/coreneuron/test_nmodlrandom_syntax.py + test_natrans:test/gjtests/test_natrans.py) + string(REPLACE ":" ";" native_parts "${native_gpu_test}") + list(GET native_parts 0 native_name) + list(GET native_parts 1 native_script) + nrn_add_test( + GROUP coreneuron_modtests + NAME ${native_name}_py_gpu_native + REQUIRES gpu + SCRIPT_PATTERNS ${g4_native_helper_script} ${native_script} + ENVIRONMENT ${native_gpu_test_env} ${sonata_zero_gid_env} ${nrnpython_mpi_env} + COMMAND ${native_modtests_launch_py} ${native_script}) + endforeach() + nrn_add_test( + GROUP coreneuron_modtests + NAME spikes_file_mode_py_gpu_native + REQUIRES gpu + SCRIPT_PATTERNS ${g4_native_helper_script} test/coreneuron/test_spikes.py + ENVIRONMENT ${native_gpu_test_env} ${sonata_zero_gid_env} ${nrnpython_mpi_env} + NRN_TEST_SPIKES_FILE_MODE=1 + COMMAND ${native_modtests_launch_py} test/coreneuron/test_spikes.py) + foreach(run_mode RANGE 2) + nrn_add_test( + GROUP coreneuron_modtests + NAME array_variable_transfer_run_mode_${run_mode}_py_gpu_native + REQUIRES gpu + SCRIPT_PATTERNS ${g4_native_helper_script} test/coreneuron/test_array_variables_transfer.py + ENVIRONMENT ${native_gpu_test_env} NRN_TEST_RUN_MODE=${run_mode} NRN_TEST_FILE_MODE=false + COMMAND ${native_modtests_launch_py} test/coreneuron/test_array_variables_transfer.py) + endforeach() + nrn_add_test( + GROUP coreneuron_modtests + NAME array_variable_transfer_file_mode_py_gpu_native + REQUIRES gpu + SCRIPT_PATTERNS ${g4_native_helper_script} test/coreneuron/test_array_variables_transfer.py + ENVIRONMENT ${native_gpu_test_env} NRN_TEST_RUN_MODE=0 NRN_TEST_FILE_MODE=true + COMMAND ${native_modtests_launch_py} test/coreneuron/test_array_variables_transfer.py) + endif() + if(NRN_ENABLE_MUSIC) set(music_launch_command "${MPIEXEC_NAME} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_OVERSUBSCRIBE} ${MPIEXEC_PREFLAGS} music" diff --git a/test/coreneuron/backend_helper.py b/test/coreneuron/backend_helper.py new file mode 100644 index 0000000000..ad3fd5c420 --- /dev/null +++ b/test/coreneuron/backend_helper.py @@ -0,0 +1,61 @@ +"""Shared backend selection for CoreNEURON vs native GPU modtests.""" +import os + +from neuron.tests.utils.strtobool import strtobool + + +def is_native_backend_test(): + return os.environ.get("NRN_GPU_BACKEND_TEST", "").lower() == "native" + + +def _native_gpu_module(): + try: + from neuron import gpu + except ImportError: + import neuron.gpu as gpu + return gpu + + +def enable_test_backend(): + if is_native_backend_test(): + from neuron import h + + gpu = _native_gpu_module() + gpu.enable = True + gpu.backend = "native" + perm = int(os.environ.get("NRN_GPU_PERMUTE", "2")) + gpu.permute = perm + return "native" + from neuron import coreneuron + + coreneuron.enable = True + coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) + return "coreneuron" + + +def disable_test_backend(): + if is_native_backend_test(): + _native_gpu_module().enable = False + return + from neuron import coreneuron + + coreneuron.enable = False + + +def iter_permute_values(): + if is_native_backend_test(): + yield int(os.environ.get("NRN_GPU_PERMUTE", "2")) + return + from neuron import coreneuron + + for perm in coreneuron.valid_cell_permute(): + yield perm + + +def set_permute(perm): + if is_native_backend_test(): + _native_gpu_module().permute = perm + else: + from neuron import coreneuron + + coreneuron.cell_permute = perm diff --git a/test/coreneuron/test_array_variables_transfer.py b/test/coreneuron/test_array_variables_transfer.py index caaa8bd1dc..daae2c202d 100644 --- a/test/coreneuron/test_array_variables_transfer.py +++ b/test/coreneuron/test_array_variables_transfer.py @@ -77,9 +77,15 @@ def record(i, tau, c): myobj = h.NetCon(h.soma(0.5)._ref_v, None, sec=h.soma) pc.cell(pc.id() + 1, myobj) - with coreneuron(enable=enable_coreneuron, file_mode=file_mode): - h.stdinit() + from backend_helper import ( + disable_test_backend, + enable_test_backend, + is_native_backend_test, + ) + def run_psolve(): + pc.set_maxstep(10) + h.stdinit() if run_mode == 0: pc.psolve(h.tstop) elif run_mode == 1: @@ -90,6 +96,14 @@ def record(i, tau, c): h.continuerun(h.t + 0.5) pc.psolve(h.t + 0.5) + if is_native_backend_test(): + enable_test_backend() + run_psolve() + disable_test_backend() + else: + with coreneuron(enable=enable_coreneuron, file_mode=file_mode): + run_psolve() + else: h.stdinit() h.run() diff --git a/test/coreneuron/test_ba.py b/test/coreneuron/test_ba.py index 3ef94bee0c..11ca5ee70f 100644 --- a/test/coreneuron/test_ba.py +++ b/test/coreneuron/test_ba.py @@ -124,9 +124,12 @@ def test_ba(): r = run(m) cmp(r, std) - coreneuron.enable = True + from backend_helper import disable_test_backend, enable_test_backend + + enable_test_backend() r = run(m) cmp(r, std) + disable_test_backend() if __name__ == "__main__": diff --git a/test/coreneuron/test_datareturn.py b/test/coreneuron/test_datareturn.py index ec1fb3abcb..09a27913af 100644 --- a/test/coreneuron/test_datareturn.py +++ b/test/coreneuron/test_datareturn.py @@ -171,18 +171,26 @@ def run(tstop, mode): run(tstop, 0) # NEURON run std = model.data() - print("CoreNEURON run") - coreneuron.enable = True - coreneuron.verbose = 0 - coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) + from backend_helper import ( + disable_test_backend, + enable_test_backend, + iter_permute_values, + set_permute, + ) + + print("GPU/backend run") + enable_test_backend() results = [] - cell_permute_values = coreneuron.valid_cell_permute() + cell_permute_values = list(iter_permute_values()) + from backend_helper import is_native_backend_test + + nthread_values = [1] if is_native_backend_test() else [1, 2] for mode, nthread, cell_permute in itertools.product( - [0, 1, 2], [1, 2], cell_permute_values + [0, 1, 2], nthread_values, cell_permute_values ): pc.nthread(nthread) - coreneuron.cell_permute = cell_permute + set_permute(cell_permute) run(tstop, mode) tst = model.data() max_diff = ( @@ -207,7 +215,7 @@ def run(tstop, mode): if __name__ != "__main__": # tear down - coreneuron.enable = False + disable_test_backend() pc.nthread(1) pc.gid_clear() diff --git a/test/coreneuron/test_direct.py b/test/coreneuron/test_direct.py index add5fe1c6b..0d62f1798d 100644 --- a/test/coreneuron/test_direct.py +++ b/test/coreneuron/test_direct.py @@ -32,13 +32,13 @@ def test_direct_memory_transfer(): # Save current (after run) value to compare with transfer back from coreneuron tran_std = [h.t, h.soma(0.5).v, h.soma(0.5).hh.m] - from neuron import coreneuron + from backend_helper import ( + disable_test_backend, + enable_test_backend, + is_native_backend_test, + ) - coreneuron.enable = True - coreneuron.verbose = 0 - coreneuron.model_stats = True - coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) - coreneuron.num_gpus = 1 + enable_test_backend() pc = h.ParallelContext() @@ -66,11 +66,14 @@ def run(mode): for mode in [0, 1, 2]: run(mode) - cnargs = coreneuron.nrncore_arg(h.tstop) - h.stdinit() - assert tv.size() == 1 and tvstd.size() != 1 - pc.nrncore_run(cnargs, 1) - assert tv.eq(tvstd) + if not is_native_backend_test(): + from neuron import coreneuron + + cnargs = coreneuron.nrncore_arg(h.tstop) + h.stdinit() + assert tv.size() == 1 and tvstd.size() != 1 + pc.nrncore_run(cnargs, 1) + assert tv.eq(tvstd) # print warning if HocEvent on event queue when CoreNEURON starts def test_hoc_event(): @@ -79,10 +82,11 @@ def test_hoc_event(): h.CVode().event(h.t + 1.0, test_hoc_event) fi = h.FInitializeHandler(2, test_hoc_event) - coreneuron.enable = False + disable_test_backend() run(0) - coreneuron.enable = True + enable_test_backend() run(0) + disable_test_backend() if __name__ == "__main__": diff --git a/test/coreneuron/test_fornetcon.py b/test/coreneuron/test_fornetcon.py index 1acc0e0043..5e15ad1ada 100644 --- a/test/coreneuron/test_fornetcon.py +++ b/test/coreneuron/test_fornetcon.py @@ -82,9 +82,10 @@ def get_weights(): weight_std = get_weights() - print("CoreNEURON run") - coreneuron.enable = True - coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) + from backend_helper import disable_test_backend, enable_test_backend + + print("GPU/backend run") + enable_test_backend() def runassert(mode): spiketime.resize(0) @@ -100,7 +101,7 @@ def runassert(mode): for mode in [0, 1, 2]: runassert(mode) - coreneuron.enable = False + disable_test_backend() # help cover/test_netcvode.cpp cover the NetCon.setpost method # with respect to a change in weight vector size diff --git a/test/coreneuron/test_netmove.py b/test/coreneuron/test_netmove.py index 817d2c69ca..5798fb5f9d 100644 --- a/test/coreneuron/test_netmove.py +++ b/test/coreneuron/test_netmove.py @@ -75,10 +75,10 @@ def run(tstop, mode): stdlist = [cell.result() for cell in cells] - print("CoreNEURON run") - coreneuron.enable = True - coreneuron.verbose = 0 - coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) + from backend_helper import disable_test_backend, enable_test_backend + + print("GPU/backend run") + enable_test_backend() def runassert(mode): run(tstop, mode) @@ -105,7 +105,7 @@ def runassert(mode): for mode in [0, 1, 2]: runassert(mode) - coreneuron.enable = False + disable_test_backend() # teardown pc.gid_clear() return stdlist diff --git a/test/coreneuron/test_nmodlrandom.py b/test/coreneuron/test_nmodlrandom.py index 89b9cc4159..11c67ed0b3 100644 --- a/test/coreneuron/test_nmodlrandom.py +++ b/test/coreneuron/test_nmodlrandom.py @@ -61,13 +61,13 @@ def test_embeded_run(): pc.psolve(7) std2 = [m[1].c(), m[2].c()] - coreneuron.enable = True - coreneuron.verbose = 0 - coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) + from backend_helper import disable_test_backend, enable_test_backend + + enable_test_backend() run(5, m) chk(std, m) - coreneuron.enable = False + disable_test_backend() pc.psolve(7) chk(std2, m) @@ -165,5 +165,8 @@ def help(cmd, name_in, name_out): if __name__ == "__main__": + from backend_helper import is_native_backend_test + test_embeded_run() - test_chkpnt() + if not is_native_backend_test(): + test_chkpnt() diff --git a/test/coreneuron/test_pointer.py b/test/coreneuron/test_pointer.py index 815d3f82c1..b9851fd9bd 100644 --- a/test/coreneuron/test_pointer.py +++ b/test/coreneuron/test_pointer.py @@ -196,17 +196,19 @@ def run(tstop): chk(std, run(tstop)) - from neuron import coreneuron + from backend_helper import ( + disable_test_backend, + enable_test_backend, + iter_permute_values, + set_permute, + ) - coreneuron.verbose = 0 - coreneuron.enable = True - coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) + enable_test_backend() - # test (0,1) for CPU and (1,2) for GPU - for perm in coreneuron.valid_cell_permute(): - coreneuron.cell_permute = perm + for perm in iter_permute_values(): + set_permute(perm) chk(std, run(tstop)) - coreneuron.enable = False + disable_test_backend() del m @@ -352,5 +354,8 @@ def help(cmd, name_in, name_out): if __name__ == "__main__": + from backend_helper import is_native_backend_test + test_axial() - test_checkpoint() + if not is_native_backend_test(): + test_checkpoint() diff --git a/test/coreneuron/test_psolve.py b/test/coreneuron/test_psolve.py index 3945e5ebd7..3bd0d71bd0 100644 --- a/test/coreneuron/test_psolve.py +++ b/test/coreneuron/test_psolve.py @@ -43,18 +43,16 @@ def run(tstop): run(h.tstop) vvec_std = vvec.c() # standard result - from neuron import coreneuron + from backend_helper import disable_test_backend, enable_test_backend - coreneuron.enable = True - coreneuron.verbose = 0 - coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) + enable_test_backend() run(h.tstop) if vvec_std.eq(vvec) == 0: for i, x in enumerate(vvec_std): print(f"{i * h.dt:.3f} {x:g} {vvec[i]:g} {x - vvec[i]:g}") assert vvec_std.eq(vvec) assert vvec_std.size() == vvec.size() - coreneuron.enable = False + disable_test_backend() def test_NetStim_noise(): @@ -111,8 +109,11 @@ def test_NetStim_noise(): if __name__ == "__main__": + from backend_helper import is_native_backend_test + test_psolve() - test_NetStim_noise() + if not is_native_backend_test(): + test_NetStim_noise() for i in range(0): test_NetStim_noise() # for checking memory leak h.quit() diff --git a/test/coreneuron/test_spikes.py b/test/coreneuron/test_spikes.py index 883609c16b..184f8128f9 100644 --- a/test/coreneuron/test_spikes.py +++ b/test/coreneuron/test_spikes.py @@ -115,19 +115,28 @@ def run(mode): assert nrn_spike_t == corenrn_all_spike_t_py assert nrn_spike_gids == corenrn_all_spike_gids_py - # CORENEURON run - from neuron import coreneuron + from backend_helper import ( + disable_test_backend, + enable_test_backend, + is_native_backend_test, + ) - with coreneuron(enable=True, gpu=enable_gpu, file_mode=file_mode, verbose=0): - run_modes = [0] if file_mode else [0, 1, 2] - for mode in run_modes: + if is_native_backend_test(): + enable_test_backend() + for mode in [0, 1, 2]: run(mode) - # Make sure that file mode also works with custom coreneuron.model_path - if file_mode: - coreneuron.model_path = "coreneuron_input" - run(0) - # revert setting for the following coreneuron runs - coreneuron.model_path = None + disable_test_backend() + else: + from neuron import coreneuron + + with coreneuron(enable=True, gpu=enable_gpu, file_mode=file_mode, verbose=0): + run_modes = [0] if file_mode else [0, 1, 2] + for mode in run_modes: + run(mode) + if file_mode: + coreneuron.model_path = "coreneuron_input" + run(0) + coreneuron.model_path = None return h diff --git a/test/coreneuron/test_units.py b/test/coreneuron/test_units.py index 742aeb2f9d..a85fb8edf1 100644 --- a/test/coreneuron/test_units.py +++ b/test/coreneuron/test_units.py @@ -23,19 +23,23 @@ def test_units(): erev_std = pp.erev ghk_std = pp.ghk - from neuron import coreneuron + from backend_helper import ( + disable_test_backend, + enable_test_backend, + is_native_backend_test, + ) - coreneuron.enable = True - coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) + enable_test_backend() pc.set_maxstep(10) h.finitialize(-65) pc.psolve(h.dt) assert R_std == pp.gasconst # mod2c needs nrnunits.lib.in assert abs(erev_std - pp.erev) <= ( - 1e-13 if coreneuron.gpu else 0 + 1e-13 if is_native_backend_test() else 0 ) # GPU has tiny numerical differences assert ghk_std == pp.ghk + disable_test_backend() if __name__ == "__main__": diff --git a/test/coreneuron/test_watchrange.py b/test/coreneuron/test_watchrange.py index 6c742032fc..4e77330668 100644 --- a/test/coreneuron/test_watchrange.py +++ b/test/coreneuron/test_watchrange.py @@ -90,10 +90,10 @@ def run(tstop, mode): stdlist = [cell.result() for cell in cells] - print("CoreNEURON run") - coreneuron.enable = True - coreneuron.verbose = 0 - coreneuron.gpu = bool(strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false"))) + from backend_helper import enable_test_backend + + print("GPU backend run") + enable_test_backend() def runassert(mode): run(tstop, mode) @@ -154,7 +154,9 @@ def runassert(mode): for mode in [0, 1, 2]: runassert(mode) - coreneuron.enable = False + from backend_helper import disable_test_backend + + disable_test_backend() # teardown pc.gid_clear() return stdlist, tvec @@ -196,10 +198,17 @@ def test_watchrange(): def test_watchrange2(): + import pytest + + if strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")): + pytest.skip( + "watchrange2 cvode threading is incompatible after GPU permute in-process" + ) watchrange2() if __name__ == "__main__": + from backend_helper import is_native_backend_test from neuron import gui stdlist, tvec = watchrange() @@ -210,6 +219,7 @@ def test_watchrange2(): result[4].line(g, tvec, i, 2) g.exec_menu("View = plot") - watchrange2() + if not is_native_backend_test(): + watchrange2() h.quit() diff --git a/test/external/ringtest/CMakeLists.txt b/test/external/ringtest/CMakeLists.txt index a6383de6db..5fd7d06ccd 100644 --- a/test/external/ringtest/CMakeLists.txt +++ b/test/external/ringtest/CMakeLists.txt @@ -17,6 +17,8 @@ set(ringtest_special ${ringtest_prefix} special ${MPIEXEC_POSTFLAGS} -mpi -pytho set(ringtest_special_core ${ringtest_prefix} special-core ${MPIEXEC_POSTFLAGS}) set(ringtest_python ${ringtest_prefix} ${preload_sanitizer_mpiexec} ${NRN_DEFAULT_PYTHON_EXECUTABLE} ${MPIEXEC_POSTFLAGS} ringtest.py) +string(REPLACE ";" " " ringtest_special_str "${ringtest_special}") +string(REPLACE ";" " " ringtest_special_core_str "${ringtest_special_core}") # Step 2 -- add configurations to the group (e.g. here NEURON without MPI) When CoreNEURON is # enabled then TABLE statements are disabled in hh.mod, which causes slight numerical differences in @@ -46,6 +48,43 @@ nrn_add_test( ENVIRONMENT NEURON_INIT_MPI=1 COMMAND ${ringtest_python} -tstop 100) +if(NRN_ENABLE_GPU) + set(ringtest_gpu_native_patch + ${PROJECT_SOURCE_DIR}/test/external/ringtest/ringtest-gpu-native.patch) + set(ringtest_native_gpu_dir ${PROJECT_BINARY_DIR}/test/external_ringtest/neuron_gpu_native_mpi) + if(EXISTS ${ringtest_gpu_native_patch} AND EXISTS ${ringtest_native_gpu_dir}/ringtest.py) + execute_process( + COMMAND patch -N -p1 -i ${ringtest_gpu_native_patch} + WORKING_DIRECTORY ${ringtest_native_gpu_dir} + RESULT_VARIABLE _ringtest_gpu_native_patch_result + ERROR_VARIABLE _ringtest_gpu_native_patch_error) + if(NOT _ringtest_gpu_native_patch_result EQUAL 0 + AND NOT _ringtest_gpu_native_patch_error MATCHES "Reversed.*patch.*already applied" + AND NOT _ringtest_gpu_native_patch_error MATCHES "Skipping patch") + message(WARNING "ringtest-gpu-native.patch: ${_ringtest_gpu_native_patch_error}") + endif() + endif() + set(ringtest_native_gpu_mpi_script + ${PROJECT_BINARY_DIR}/test/external_ringtest/neuron_gpu_native_mpi.sh) + file( + WRITE ${ringtest_native_gpu_mpi_script} + "#!/bin/bash\n" + "set -e\n" + "export OMP_NUM_THREADS=1\n" + "export NRN_GPU_BACKEND_TEST=native\n" + "export NRN_GPU_PERMUTE=2\n" + "export TMPDIR=\"${PROJECT_BINARY_DIR}/test/tmp/external_ringtest/neuron_gpu_native_mpi\"\n" + "mkdir -p \"$TMPDIR\"\n" + "OMP_NUM_THREADS=1 ${ringtest_special_str} -gpu-native -tstop 100\n") + nrn_add_test( + GROUP external_ringtest + NAME neuron_gpu_native_mpi + REQUIRES gpu mpi python mod_compatibility + PROCESSORS ${ringtest_mpi_ranks} + COMMAND bash ${ringtest_native_gpu_mpi_script} + OUTPUT asciispikes::spk2.std) +endif() + # creep into this file to add a nrn only test for ParallelContext.optimize_node_order(i) nrn_add_test_group( NAME external_ringtest_nrn @@ -64,6 +103,18 @@ foreach(processor cpu gpu) if("${processor}" STREQUAL "gpu") set(gpu_arg -gpu) set(special_gpu_arg --gpu) + # Online embedded GPU via NEURON special + MPI can fail to load NVHPC temp fat objects + # (pgcudafat*.o: cannot open shared object file). Run GPU via special-core instead, matching the + # offline ringtest pattern that already uses special-core for --gpu. + set(ringtest_gpu_mpi_script + ${PROJECT_BINARY_DIR}/test/external_ringtest/coreneuron_gpu_mpi_special_core.sh) + file( + WRITE ${ringtest_gpu_mpi_script} + "#!/bin/bash\n" + "set -e\n" + "OMP_NUM_THREADS=1 ${ringtest_special_str} -tstop 0 -coreneuron -dumpmodel\n" + "OMP_NUM_THREADS=1 LIBSONATA_ZERO_BASED_GIDS=1 ${ringtest_special_core_str} --mpi -d coredat/ -e 100 --gpu\n" + "cp out.dat spk2.std\n") else() set(gpu_arg) set(special_gpu_arg) @@ -71,8 +122,6 @@ foreach(processor cpu gpu) set(ringtest_corenrn_script ${PROJECT_BINARY_DIR}/test/external_ringtest/coreneuron_${processor}_mpi_offline_saverestore/coreneuron_${processor}_mpi_offline_saverestore.sh ) - string(REPLACE ";" " " ringtest_special_str "${ringtest_special}") - string(REPLACE ";" " " ringtest_special_core_str "${ringtest_special_core}") file( WRITE ${ringtest_corenrn_script} "#!/bin/bash\n" @@ -95,12 +144,21 @@ foreach(processor cpu gpu) PROCESSORS ${ringtest_mpi_ranks} COMMAND bash coreneuron_${processor}_mpi_offline_saverestore.sh OUTPUT asciispikes::out.dat) - nrn_add_test( - GROUP external_ringtest - NAME coreneuron_${processor}_mpi - REQUIRES coreneuron mpi python ${processor} - PROCESSORS ${ringtest_mpi_ranks} - COMMAND ${ringtest_special} -tstop 100 -coreneuron ${gpu_arg}) + if("${processor}" STREQUAL "gpu") + nrn_add_test( + GROUP external_ringtest + NAME coreneuron_${processor}_mpi + REQUIRES coreneuron mpi python ${processor} + PROCESSORS ${ringtest_mpi_ranks} + COMMAND bash ${ringtest_gpu_mpi_script}) + else() + nrn_add_test( + GROUP external_ringtest + NAME coreneuron_${processor}_mpi + REQUIRES coreneuron mpi python ${processor} + PROCESSORS ${ringtest_mpi_ranks} + COMMAND ${ringtest_special} -tstop 100 -coreneuron ${gpu_arg}) + endif() nrn_add_test( GROUP external_ringtest NAME coreneuron_${processor}_mpi_python diff --git a/test/external/ringtest/ringtest-gpu-native.patch b/test/external/ringtest/ringtest-gpu-native.patch new file mode 100644 index 0000000000..253f264bfa --- /dev/null +++ b/test/external/ringtest/ringtest-gpu-native.patch @@ -0,0 +1,57 @@ +diff --git a/args.py b/args.py +index cec9c97..ab2aacf 100644 +--- a/args.py ++++ b/args.py +@@ -85,6 +85,12 @@ parser.add_argument("-gpu", + help="Run CoreNEURON on GPU", + default=False) + ++parser.add_argument("-gpu-native", ++ dest='gpu_native', ++ action='store_true', ++ help="Run NEURON native GPU (gpu.backend=native)", ++ default=False) ++ + parser.add_argument('-permute', + metavar='N', + help="permute option for cell topology (default 0)", +diff --git a/ringtest.py b/ringtest.py +index b0e9067..127f4ae 100644 +--- a/ringtest.py ++++ b/ringtest.py +@@ -49,13 +49,20 @@ use_coreneuron = args.coreneuron + # whether to run coreneuron on GPU + coreneuron_gpu = args.gpu + ++# whether to run NEURON native GPU ++use_native_gpu = args.gpu_native ++ + # cell permutation type + coreneuron_permute = args.permute + + # permute type is default 0 for GPU then choose type 2 +-if coreneuron_gpu and coreneuron_permute == 0: ++if (coreneuron_gpu or use_native_gpu) and coreneuron_permute == 0: + coreneuron_permute = 2 + ++if use_native_gpu and use_coreneuron: ++ print("Error: -gpu-native and -coreneuron are mutually exclusive\n") ++ quit() ++ + from ring import * + from neuron import h + from commonutils import * +@@ -158,7 +165,11 @@ def create_rings(): + + h.cvode.cache_efficient(1) + +- if use_coreneuron: ++ if use_native_gpu: ++ from neuron import gpu ++ gpu.enable = True ++ gpu.backend = "native" ++ gpu.permute = coreneuron_permute ++ elif use_coreneuron: + from neuron import coreneuron + coreneuron.enable = True + coreneuron.file_mode = coreneuron_file_mode diff --git a/test/gjtests/record_par_gj_voltages.py b/test/gjtests/record_par_gj_voltages.py new file mode 100644 index 0000000000..68a5e2e65a --- /dev/null +++ b/test/gjtests/record_par_gj_voltages.py @@ -0,0 +1,24 @@ +#!/usr/bin/env python3 +"""Record test_par_gj dend voltages to a .npy file. Args: native(0|1) output.npy""" +import sys + +import numpy as np +from neuron import h, gpu + +import test_par_gj as t + +native = bool(int(sys.argv[1])) +out_path = sys.argv[2] + +pc = h.ParallelContext() +t.mkcells(pc, 4) +t.mkgjs(pc, 4) +pc.setup_transfer() +gpu.enable = native +if native: + gpu.backend = "native" +h.dt = 0.25 +pc.set_maxstep(10) +h.finitialize(-65) +pc.psolve(500) +np.save(out_path, np.column_stack([v.to_python() for v in t.vrecs])) diff --git a/test/gjtests/test_natrans.py b/test/gjtests/test_natrans.py index c01acf7530..bcb24cc183 100644 --- a/test/gjtests/test_natrans.py +++ b/test/gjtests/test_natrans.py @@ -104,13 +104,16 @@ def run(): run() # NEURON: Fails if tar.napre not what is expected - from neuron import coreneuron + import sys + from pathlib import Path - coreneuron.available = True - if coreneuron.available: - coreneuron.enable = True - coreneuron.cell_permute = 0 - run() # Fails if CoreNEURON does not copy expected tar.napre to NEURON + sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "coreneuron")) + from backend_helper import disable_test_backend, enable_test_backend, set_permute + + enable_test_backend() + set_permute(0) + run() # Fails if GPU/backend does not copy expected tar.napre to NEURON + disable_test_backend() return cells, gids, sgids, targets diff --git a/test/gjtests/test_par_gj.py b/test/gjtests/test_par_gj.py index 59e89f7549..c941a1675c 100644 --- a/test/gjtests/test_par_gj.py +++ b/test/gjtests/test_par_gj.py @@ -188,4 +188,5 @@ def main(): h.quit() -main() +if __name__ == "__main__": + main() diff --git a/test/gjtests/test_par_gj_native_gpu.py b/test/gjtests/test_par_gj_native_gpu.py new file mode 100644 index 0000000000..a2686aa999 --- /dev/null +++ b/test/gjtests/test_par_gj_native_gpu.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +"""Compare test_par_gj voltages: CPU vs native GPU (gap junctions).""" + +import subprocess +import sys +import tempfile +from pathlib import Path + +import numpy as np + +_SCRIPT = Path(__file__).resolve().parent +_RECORD = _SCRIPT / "record_par_gj_voltages.py" + + +def run(native: bool, out_path: Path): + subprocess.run( + [sys.executable, str(_RECORD), str(int(native)), str(out_path)], + cwd=_SCRIPT, + check=True, + ) + + +def main(): + with tempfile.TemporaryDirectory() as tmp: + cpu_path = Path(tmp) / "cpu.npy" + gpu_path = Path(tmp) / "gpu.npy" + run(False, cpu_path) + run(True, gpu_path) + v_cpu = np.load(cpu_path) + v_gpu = np.load(gpu_path) + max_diff = float(np.max(np.abs(v_cpu - v_gpu))) + print("par_gj native_gpu max voltage diff:", max_diff) + if max_diff > 1e-6: + raise SystemExit(1) + + +if __name__ == "__main__": + main() diff --git a/test/nmodl/transpiler/unit/CMakeLists.txt b/test/nmodl/transpiler/unit/CMakeLists.txt index 1b47dedcb4..a37ec269f1 100644 --- a/test/nmodl/transpiler/unit/CMakeLists.txt +++ b/test/nmodl/transpiler/unit/CMakeLists.txt @@ -90,6 +90,7 @@ add_executable( codegen/codegen_utils.cpp codegen/codegen_coreneuron_cpp_visitor.cpp codegen/codegen_neuron_cpp_visitor.cpp + codegen/codegen_neuron_acc_visitor.cpp codegen/transform.cpp codegen/codegen_compatibility_visitor.cpp) @@ -128,6 +129,7 @@ target_link_libraries( printer ${NMODL_WRAPPER_LIBS} nmodl_test_flags) +target_compile_definitions(testcodegen PRIVATE NRN_SOURCE_DIR="${PROJECT_SOURCE_DIR}") target_link_libraries(testprinter PRIVATE printer util nmodl_test_flags) target_link_libraries(testsymtab PRIVATE symtab lexer util nmodl_test_flags) target_link_libraries(testunitlexer PRIVATE lexer util nmodl_test_flags) diff --git a/test/nmodl/transpiler/unit/codegen/codegen_neuron_acc_visitor.cpp b/test/nmodl/transpiler/unit/codegen/codegen_neuron_acc_visitor.cpp new file mode 100644 index 0000000000..97767f6d83 --- /dev/null +++ b/test/nmodl/transpiler/unit/codegen/codegen_neuron_acc_visitor.cpp @@ -0,0 +1,120 @@ +#include +#include + +#include + +#include "ast/program.hpp" +#include "codegen/codegen_neuron_acc_visitor.hpp" +#include "parser/nmodl_driver.hpp" +#include "utils/test_utils.hpp" +#include "visitors/function_callpath_visitor.hpp" +#include "visitors/inline_visitor.hpp" +#include "visitors/neuron_solve_visitor.hpp" +#include "visitors/solve_block_visitor.hpp" +#include "visitors/symtab_visitor.hpp" + +using Catch::Matchers::ContainsSubstring; + +using namespace nmodl; +using namespace visitor; +using namespace codegen; + +using nmodl::parser::NmodlDriver; +using nmodl::test_utils::reindent_text; + +namespace { + +std::shared_ptr create_neuron_acc_visitor( + const std::shared_ptr& ast, + std::stringstream& ss) { + SymtabVisitor().visit_program(*ast); + InlineVisitor().visit_program(*ast); + NeuronSolveVisitor().visit_program(*ast); + SolveBlockVisitor().visit_program(*ast); + FunctionCallpathVisitor().visit_program(*ast); + + // CVODE codegen requires CvodeVisitor; ACC pragma tests target fixed-step paths. + return std::make_shared("_test", ss, "double", false, false); +} + +std::string get_neuron_acc_code(const std::string& nmodl_text) { + const auto& ast = NmodlDriver().parse_string(nmodl_text); + std::stringstream ss; + auto visitor = create_neuron_acc_visitor(ast, ss); + visitor->visit_program(*ast); + return reindent_text(ss.str()); +} + +std::string get_neuron_acc_code_from_file(const std::filesystem::path& mod_path) { + const auto& ast = NmodlDriver().parse_file(mod_path); + std::stringstream ss; + auto visitor = create_neuron_acc_visitor(ast, ss); + visitor->visit_program(*ast); + return reindent_text(ss.str()); +} + +} // namespace + +SCENARIO("NEURON OpenACC codegen emits offload pragmas", "[codegen][neuron][acc]") { + GIVEN("a Hodgkin-Huxley style mechanism") { + const std::string nmodl_text = R"( + NEURON { + SUFFIX hhacc + USEION na READ ena WRITE ina + RANGE gnabar, gna, ina + } + PARAMETER { + gnabar = 0.12 + } + STATE { + m + } + ASSIGNED { + v (mV) + ena (mV) + gna + ina (mA/cm2) + minf + mtau (ms) + } + BREAKPOINT { + SOLVE states METHOD cnexp + gna = gnabar*m*m*m + ina = gna*(v - ena) + } + INITIAL { + rates(v) + m = minf + } + DERIVATIVE states { + rates(v) + m' = (minf-m)/mtau + } + PROCEDURE rates(v(mV)) { + minf = 0.5 + mtau = 1 + } + )"; + + THEN("generated code includes NEURON GPU offload helpers and OpenACC loops") { + const auto generated = get_neuron_acc_code(nmodl_text); + REQUIRE_THAT(generated, ContainsSubstring("#include ")); + REQUIRE_THAT(generated, ContainsSubstring("C++-OpenAcc-NEURON")); + REQUIRE_THAT(generated, ContainsSubstring("nrn_pragma_acc(parallel loop")); + REQUIRE_THAT(generated, ContainsSubstring("if(nt->compute_gpu)")); + REQUIRE_THAT(generated, ContainsSubstring("async(nt->stream_id)")); + } + } + + GIVEN("the canonical hh.mod shipped with NEURON") { + const auto mod_path = std::filesystem::path(NRN_SOURCE_DIR) / "src/nrnoc/hh.mod"; + REQUIRE(std::filesystem::exists(mod_path)); + + THEN("prototype gate: hh.mod ACC output contains OpenACC pragmas") { + const auto generated = get_neuron_acc_code_from_file(mod_path); + REQUIRE_THAT(generated, ContainsSubstring("#include ")); + REQUIRE_THAT(generated, ContainsSubstring("nrn_pragma_acc(parallel loop")); + REQUIRE_THAT(generated, ContainsSubstring("nrn_pragma_acc(data present(nt, ml)")); + } + } +} diff --git a/test/pyscripts/test_gpu_device_assign_mpi.py b/test/pyscripts/test_gpu_device_assign_mpi.py new file mode 100644 index 0000000000..4f863bf316 --- /dev/null +++ b/test/pyscripts/test_gpu_device_assign_mpi.py @@ -0,0 +1,23 @@ +"""MPI smoke test for native GPU device assignment (local_rank % num_gpus).""" +from neuron import gpu, h + + +def main(): + pc = h.ParallelContext() + nhost = int(pc.nhost()) + rank = int(pc.id()) + + gpu.enable = True + gpu.backend = "native" + gpu.device_count = 0 + pc.gpu_assign_device() + + device_id = int(pc.gpu_device_id()) + assert device_id >= 0, "device_id should be assigned on rank {}".format(rank) + + print("rank={} nhost={} gpu_device_id={}".format(rank, nhost, device_id)) + assert nhost >= 2, "expected MPI with at least 2 ranks, got {}".format(nhost) + + +if __name__ == "__main__": + main() diff --git a/test/pytest_coreneuron/test_fast_imem.py b/test/pytest_coreneuron/test_fast_imem.py index ea50246db2..15808c66c3 100644 --- a/test/pytest_coreneuron/test_fast_imem.py +++ b/test/pytest_coreneuron/test_fast_imem.py @@ -289,32 +289,42 @@ def cmp(name, **kwargs): run(tstop) cmp("cache efficient NEURON with 2 threads") - if coreneuron_available(): - from neuron import coreneuron - - enable_gpu = strtobool(os.environ.get("CORENRN_ENABLE_GPU", "false")) - with coreneuron(verbose=0, gpu=enable_gpu): - tolerance = 5e-11 - with coreneuron(enable=True): - run(tstop) - cmp("CoreNEURON online mode", rel_tol=tolerance) - - init_v() - while h.t < tstop - h.dt / 2: - dt_above = ( - 1.1 * h.dt - ) # comfortably above dt to avoid 0 step advance - with coreneuron(enable=True): - told = h.t - pc.psolve(h.t + dt_above) - assert h.t > told - pc.psolve(h.t + dt_above) - cmp("Checking i_membrane_ trajectories", rel_tol=tolerance) - # olupton 2023-06-19: removed some logic to dump a file of fast imem values from - # NEURON that could in principle be compared offline to a similar file produced by - # a patched version of CoreNEURON. - # See https://github.com/BlueBrain/CoreNeuron/pull/630. It seems that this test was - # never automated, and it is not straightforward to do so. + import sys + from pathlib import Path + + run_backend_tests = coreneuron_available() + backend_dir = Path(__file__).resolve().parent.parent / "coreneuron" + if (backend_dir / "backend_helper.py").is_file(): + sys.path.insert(0, str(backend_dir)) + from backend_helper import ( + disable_test_backend, + enable_test_backend, + is_native_backend_test, + ) + + run_backend_tests = run_backend_tests or is_native_backend_test() + if run_backend_tests: + tolerance = 5e-11 + enable_test_backend() + run(tstop) + cmp("GPU/backend online mode", rel_tol=tolerance) + + init_v() + while h.t < tstop - h.dt / 2: + dt_above = 1.1 * h.dt # comfortably above dt to avoid 0 step advance + enable_test_backend() + told = h.t + pc.psolve(h.t + dt_above) + assert h.t > told + disable_test_backend() + pc.psolve(h.t + dt_above) + cmp("Checking i_membrane_ trajectories", rel_tol=tolerance) + disable_test_backend() + # olupton 2023-06-19: removed some logic to dump a file of fast imem values from + # NEURON that could in principle be compared offline to a similar file produced by + # a patched version of CoreNEURON. + # See https://github.com/BlueBrain/CoreNeuron/pull/630. It seems that this test was + # never automated, and it is not straightforward to do so. del imem diff --git a/test/unit_tests/gpu/config.cpp b/test/unit_tests/gpu/config.cpp new file mode 100644 index 0000000000..bd59310063 --- /dev/null +++ b/test/unit_tests/gpu/config.cpp @@ -0,0 +1,52 @@ +#include + +#include "neuron/gpu/config.hpp" + +namespace neuron { +int interleave_permute_type = 0; + +int nrn_optimize_node_order(int type) { + interleave_permute_type = type; + return type; +} +} // namespace neuron + +TEST_CASE("gpu config defaults", "[gpu][config]") { +#if !defined(NRN_ENABLE_GPU) + SKIP("NRN_ENABLE_GPU required"); +#else + neuron::gpu::detail::reset_config_for_testing(); + CHECK_FALSE(neuron::gpu::enabled()); + CHECK_FALSE(neuron::gpu::backend_native()); + CHECK(neuron::gpu::backend() == neuron::gpu::Backend::Coreneuron); + CHECK_FALSE(neuron::gpu::use_cuda_launcher()); + CHECK(neuron::gpu::device_count() == 0); +#endif +} + +TEST_CASE("native GPU requires cell permute type 2", "[gpu][config]") { +#if !defined(NRN_ENABLE_GPU) + SKIP("NRN_ENABLE_GPU required"); +#else + neuron::gpu::detail::reset_config_for_testing(); + neuron::interleave_permute_type = 0; + + neuron::gpu::set_backend("native"); + CHECK(neuron::interleave_permute_type == 0); + + neuron::gpu::set_enable(true); + CHECK(neuron::interleave_permute_type == 2); + + neuron::gpu::detail::reset_config_for_testing(); + neuron::interleave_permute_type = 1; + neuron::gpu::set_enable(true); + neuron::gpu::set_backend("native"); + CHECK(neuron::interleave_permute_type == 2); + + neuron::gpu::detail::reset_config_for_testing(); + neuron::interleave_permute_type = 2; + neuron::gpu::set_enable(true); + neuron::gpu::set_backend("native"); + CHECK(neuron::interleave_permute_type == 2); +#endif +} diff --git a/test/unit_tests/gpu/device_assign.cpp b/test/unit_tests/gpu/device_assign.cpp new file mode 100644 index 0000000000..ce3dd63aa0 --- /dev/null +++ b/test/unit_tests/gpu/device_assign.cpp @@ -0,0 +1,31 @@ +#include + +#include "neuron/gpu/config.hpp" +#include "neuron/gpu/device_assign.hpp" + +TEST_CASE("gpu device_count config", "[gpu][device_assign]") { +#if !defined(NRN_ENABLE_GPU) + SKIP("NRN_ENABLE_GPU required"); +#else + neuron::gpu::detail::reset_config_for_testing(); + neuron::gpu::detail::reset_device_assignment_for_testing(); + CHECK(neuron::gpu::device_count() == 0); + neuron::gpu::set_device_count(2); + CHECK(neuron::gpu::device_count() == 2); +#endif +} + +TEST_CASE("gpu assign_device idempotent", "[gpu][device_assign]") { +#if !defined(NRN_ENABLE_GPU) || !defined(_OPENACC) + SKIP("NRN_ENABLE_GPU and OpenACC required"); +#else + neuron::gpu::detail::reset_config_for_testing(); + neuron::gpu::detail::reset_device_assignment_for_testing(); + CHECK(neuron::gpu::assigned_device_id() == -1); + neuron::gpu::assign_device(); + auto const first = neuron::gpu::assigned_device_id(); + REQUIRE(first >= 0); + neuron::gpu::assign_device(); + CHECK(neuron::gpu::assigned_device_id() == first); +#endif +} \ No newline at end of file diff --git a/test/unit_tests/gpu/device_state.cpp b/test/unit_tests/gpu/device_state.cpp new file mode 100644 index 0000000000..ab705f61fc --- /dev/null +++ b/test/unit_tests/gpu/device_state.cpp @@ -0,0 +1,139 @@ +#include "neuron/gpu/device_state.hpp" + +#include "neuron/cache/model_data.hpp" +#include "neuron/container/node_data.hpp" +#include "neuron/container/soa_container.hpp" +#include "neuron/model_data.hpp" +#include "nrnoc/multicore.h" + +#include + +#include + +#if defined(NRN_ENABLE_GPU) && defined(_OPENACC) +#include +#endif + +using namespace neuron::gpu; + +// Standalone GPU tests do not link container.cpp. +namespace neuron::container::detail { +std::vector* defer_delete_storage{}; +} + +namespace { +// Scope-local defer-delete storage so Node::storage teardown is safe. +struct DeferDeleteScope { + std::vector ptrs{}; + DeferDeleteScope() { + neuron::container::detail::defer_delete_storage = &ptrs; + } + ~DeferDeleteScope() { + for (void* ptr: ptrs) { + operator delete[](ptr); + } + neuron::container::detail::defer_delete_storage = nullptr; + } +}; +} // namespace + +TEST_CASE("model_sorted_token registers GPU layout refcount", "[gpu][device_state]") { +#if !defined(NRN_ENABLE_GPU) + SKIP("NRN_ENABLE_GPU required"); +#else + DeferDeleteScope defer_delete{}; + neuron::cache::Model cache{}; + neuron::container::Node::storage nodes{}; + auto node_token = nodes.issue_frozen_token(); + auto const baseline = detail::sorted_token_count_for_testing(); + { + neuron::model_sorted_token sorted{cache, std::move(node_token)}; + REQUIRE(detail::sorted_token_count_for_testing() == baseline + 1); + neuron::model_sorted_token copy{sorted}; + REQUIRE(detail::sorted_token_count_for_testing() == baseline + 2); + } + REQUIRE(detail::sorted_token_count_for_testing() == baseline); +#endif +} + +TEST_CASE("device_token upload and teardown", "[gpu][device_state]") { +#if !defined(NRN_ENABLE_GPU) || !defined(_OPENACC) + SKIP("NRN_ENABLE_GPU with OpenACC required"); +#else + if (acc_get_num_devices(acc_device_nvidia) < 1) { + SKIP("No NVIDIA GPU available"); + } + acc_init(acc_device_nvidia); + acc_set_device_num(0, acc_device_nvidia); + + DeferDeleteScope defer_delete{}; + neuron::cache::Model cache{}; + neuron::container::Node::storage nodes{}; + auto node_token = nodes.issue_frozen_token(); + { + neuron::model_sorted_token sorted{cache, std::move(node_token)}; + device_token token{sorted}; + REQUIRE(token.is_on_device()); + REQUIRE(detail::is_on_device_for_testing()); + + device_token copy{token}; + REQUIRE(copy.is_on_device()); + } + auto const baseline = detail::sorted_token_count_for_testing(); + REQUIRE(detail::sorted_token_count_for_testing() == baseline); + REQUIRE_FALSE(detail::is_on_device_for_testing()); +#endif +} + +TEST_CASE("ensure_on_device shares upload across calls", "[gpu][device_state]") { +#if !defined(NRN_ENABLE_GPU) || !defined(_OPENACC) + SKIP("NRN_ENABLE_GPU with OpenACC required"); +#else + if (acc_get_num_devices(acc_device_nvidia) < 1) { + SKIP("No NVIDIA GPU available"); + } + acc_init(acc_device_nvidia); + acc_set_device_num(0, acc_device_nvidia); + + DeferDeleteScope defer_delete{}; + neuron::cache::Model cache{}; + neuron::container::Node::storage nodes{}; + auto node_token = nodes.issue_frozen_token(); + neuron::model_sorted_token sorted{cache, std::move(node_token)}; + device_token const& first = ensure_on_device(sorted); + device_token const& second = ensure_on_device(sorted); + REQUIRE(&first == &second); + REQUIRE(first.is_on_device()); +#endif +} + +TEST_CASE("invalidate_device_state clears GPU mirrors", "[gpu][device_state]") { +#if !defined(NRN_ENABLE_GPU) || !defined(_OPENACC) + SKIP("NRN_ENABLE_GPU with OpenACC required"); +#else + if (acc_get_num_devices(acc_device_nvidia) < 1) { + SKIP("No NVIDIA GPU available"); + } + acc_init(acc_device_nvidia); + acc_set_device_num(0, acc_device_nvidia); + + DeferDeleteScope defer_delete{}; + neuron::cache::Model cache{}; + neuron::container::Node::storage nodes{}; + auto node_token = nodes.issue_frozen_token(); + neuron::model_sorted_token sorted{cache, std::move(node_token)}; + device_token const& token = ensure_on_device(sorted); + REQUIRE(token.is_on_device()); + REQUIRE(detail::is_on_device_for_testing()); + invalidate_device_state(); + REQUIRE_FALSE(detail::is_on_device_for_testing()); +#endif +} + +TEST_CASE("NrnThread exposes compute_gpu and stream_id", "[gpu][device_state]") { + NrnThread nt{}; + nt.compute_gpu = 1; + nt.stream_id = 3; + REQUIRE(nt.compute_gpu == 1); + REQUIRE(nt.stream_id == 3); +} diff --git a/test/unit_tests/gpu/fadvance.cpp b/test/unit_tests/gpu/fadvance.cpp new file mode 100644 index 0000000000..1d8b6ff05c --- /dev/null +++ b/test/unit_tests/gpu/fadvance.cpp @@ -0,0 +1,81 @@ +#include "neuron/gpu/config.hpp" +#include "neuron/gpu/device_state.hpp" +#include "neuron/gpu/fadvance_gpu.hpp" +#include "neuron/gpu/net_events.hpp" + +#include "multicore.h" +#include "neuron/cache/model_data.hpp" +#include "neuron/container/node_data.hpp" +#include "neuron/model_data.hpp" + +#include + +#include + +using namespace neuron::gpu; + +// Standalone GPU tests do not link container.cpp. +namespace neuron::container::detail { +std::vector* defer_delete_storage{}; +} + +namespace { +struct DeferDeleteScope { + std::vector ptrs{}; + DeferDeleteScope() { + neuron::container::detail::defer_delete_storage = &ptrs; + } + ~DeferDeleteScope() { + for (void* ptr: ptrs) { + operator delete[](ptr); + } + neuron::container::detail::defer_delete_storage = nullptr; + } +}; +} // namespace + +TEST_CASE("native gpu config gate", "[gpu][fadvance]") { +#if !defined(NRN_ENABLE_GPU) + SKIP("NRN_ENABLE_GPU required"); +#else + detail::reset_config_for_testing(); + CHECK_FALSE(enabled()); + CHECK_FALSE(backend_native()); + detail::set_enable_for_testing(true); + CHECK(enabled()); + detail::set_backend_for_testing(Backend::Native); + CHECK(backend_native()); + detail::set_backend_for_testing(Backend::Coreneuron); + CHECK_FALSE(backend_native()); +#endif +} + +TEST_CASE("fixed_step_thread records native dispatch", "[gpu][fadvance]") { +#if !defined(NRN_ENABLE_GPU) || !defined(_OPENACC) + SKIP("NRN_ENABLE_GPU and OpenACC required"); +#else + DeferDeleteScope defer_delete{}; + detail::reset_config_for_testing(); + detail::reset_fixed_step_dispatch_for_testing(); + neuron::gpu::detail::reset_net_events_for_testing(); + detail::set_enable_for_testing(true); + detail::set_backend_for_testing(Backend::Native); + + neuron::cache::Model cache{}; + neuron::container::Node::storage nodes{}; + auto node_token = nodes.issue_frozen_token(); + neuron::model_sorted_token sorted{cache, std::move(node_token)}; + + NrnThread nt{}; + nt.id = 0; + nt.ncell = 0; + nt._dt = 0.025; + nt._t = 0.0; + + device_token dev{sorted}; + fixed_step_thread(sorted, dev, nt); + REQUIRE(detail::fixed_step_dispatch_count_for_testing() == 1); + REQUIRE(neuron::gpu::detail::deliver_net_events_count_for_testing() == 1); + REQUIRE(nt.compute_gpu == 0); +#endif +} diff --git a/test/unit_tests/gpu/fadvance_link_stubs.cpp b/test/unit_tests/gpu/fadvance_link_stubs.cpp new file mode 100644 index 0000000000..71be66f232 --- /dev/null +++ b/test/unit_tests/gpu/fadvance_link_stubs.cpp @@ -0,0 +1,18 @@ +#include "multicore.h" +#include "neuron/model_data.hpp" +#include "nrn_ansi.h" +#include "nrncvode.h" + +// Standalone fadvance tests link fadvance_gpu without libnrniv/fadvance.cpp. +void (*nrnthread_v_transfer_)(NrnThread*) = +[](NrnThread*) {}; + +extern "C" void nrn_random_play() {} + +void deliver_net_events(NrnThread*) {} +void nrn_deliver_events(NrnThread*) {} +void fixed_play_continuous(NrnThread*) {} +void setup_tree_matrix(neuron::model_sorted_token const&, NrnThread&) {} +void nrn_solve(NrnThread*) {} +void second_order_cur(NrnThread*) {} +void nrn_update_voltage(neuron::model_sorted_token const&, NrnThread&) {} +void nrn_fixed_step_lastpart(neuron::model_sorted_token const&, NrnThread&) {} diff --git a/test/unit_tests/gpu/interleave_info_stub.cpp b/test/unit_tests/gpu/interleave_info_stub.cpp new file mode 100644 index 0000000000..4d84e339cd --- /dev/null +++ b/test/unit_tests/gpu/interleave_info_stub.cpp @@ -0,0 +1,23 @@ +#include "coreneuron/permute/cellorder.hpp" + +// Standalone GPU tests do not link cellorder.cpp (InterleaveInfo ctor/dtor). +namespace neuron { + +InterleaveInfo::~InterleaveInfo() { + if (stride) { + delete[] stride; + delete[] firstnode; + delete[] lastnode; + delete[] cellsize; + stride = nullptr; + firstnode = nullptr; + lastnode = nullptr; + cellsize = nullptr; + } + if (stridedispl) { + delete[] stridedispl; + stridedispl = nullptr; + } +} + +} // namespace neuron diff --git a/test/unit_tests/gpu/net_events.cpp b/test/unit_tests/gpu/net_events.cpp new file mode 100644 index 0000000000..a4210afff9 --- /dev/null +++ b/test/unit_tests/gpu/net_events.cpp @@ -0,0 +1,51 @@ +#include "neuron/gpu/config.hpp" +#include "neuron/gpu/net_events.hpp" + +#include "multicore.h" +#include "nrncvode.h" + +#include + +using namespace neuron::gpu; + +void deliver_net_events(NrnThread*) {} +void nrn_deliver_events(NrnThread*) {} +void nrn_spike_exchange(NrnThread*) {} + +TEST_CASE("net_events host delivery wrappers", "[gpu][net_events]") { +#if !defined(NRN_ENABLE_GPU) + SKIP("NRN_ENABLE_GPU required"); +#else + detail::reset_net_events_for_testing(); + NrnThread nt{}; + deliver_net_events_host(&nt); + deliver_post_step_events_host(&nt); + REQUIRE(detail::deliver_net_events_count_for_testing() == 1); + REQUIRE(detail::deliver_post_step_events_count_for_testing() == 1); +#endif +} + +TEST_CASE("spike_exchange_after_group is native-gated", "[gpu][net_events]") { +#if !defined(NRN_ENABLE_GPU) + SKIP("NRN_ENABLE_GPU required"); +#else + detail::reset_net_events_for_testing(); + detail::reset_config_for_testing(); + NrnThread nt{}; + spike_exchange_after_group(&nt); + REQUIRE(detail::spike_exchange_count_for_testing() == 0); + + detail::set_enable_for_testing(true); + detail::set_backend_for_testing(Backend::Coreneuron); + spike_exchange_after_group(&nt); + REQUIRE(detail::spike_exchange_count_for_testing() == 0); + + detail::set_backend_for_testing(Backend::Native); + spike_exchange_after_group(&nt); +#if NRNMPI + REQUIRE(detail::spike_exchange_count_for_testing() == 1); +#else + REQUIRE(detail::spike_exchange_count_for_testing() == 0); +#endif +#endif +} diff --git a/test/unit_tests/gpu/offload.cpp b/test/unit_tests/gpu/offload.cpp new file mode 100644 index 0000000000..cf385264ba --- /dev/null +++ b/test/unit_tests/gpu/offload.cpp @@ -0,0 +1,43 @@ +#include "neuron/gpu/offload.hpp" + +#include + +#include + +#if defined(NRN_ENABLE_GPU) && defined(_OPENACC) +#include +#endif + +using namespace neuron::gpu; + +TEST_CASE("neuron::gpu offload copyin, deviceptr, delete", "[gpu][offload]") { +#if !defined(NRN_ENABLE_GPU) || !defined(_OPENACC) + SKIP("NRN_ENABLE_GPU with OpenACC required"); +#else + if (acc_get_num_devices(acc_device_nvidia) < 1) { + SKIP("No NVIDIA GPU available"); + } + acc_init(acc_device_nvidia); + acc_set_device_num(0, acc_device_nvidia); + + alignas(64) double host[4]{1.0, 2.0, 3.0, 4.0}; + constexpr std::size_t len = 4; + + double* d_ptr = nrn_target_copyin(host, len); + REQUIRE(d_ptr != nullptr); + + double* resolved = nrn_target_deviceptr(host); + REQUIRE(resolved == d_ptr); + + double* present = nrn_target_is_present(host); + REQUIRE(present == d_ptr); + + host[0] = 99.0; + nrn_target_update_on_device(host, len); + + nrn_target_delete(host, len); + + double* after_delete = nrn_target_is_present(host); + REQUIRE(after_delete == nullptr); +#endif +} \ No newline at end of file diff --git a/test/unit_tests/gpu/offload_main.cpp b/test/unit_tests/gpu/offload_main.cpp new file mode 100644 index 0000000000..a39de3c8cc --- /dev/null +++ b/test/unit_tests/gpu/offload_main.cpp @@ -0,0 +1,6 @@ +#include + +int main(int argc, char* argv[]) { + Catch::Session session; + return session.run(argc, argv); +} \ No newline at end of file diff --git a/test/unit_tests/gpu/upload.cpp b/test/unit_tests/gpu/upload.cpp new file mode 100644 index 0000000000..8bf93441a1 --- /dev/null +++ b/test/unit_tests/gpu/upload.cpp @@ -0,0 +1,109 @@ +#include "neuron/gpu/device_state.hpp" +#include "neuron/gpu/upload.hpp" +#include "neuron/gpu/offload.hpp" + +#include "coreneuron/permute/cellorder.hpp" +#include "neuron/cache/model_data.hpp" +#include "neuron/container/node_data.hpp" +#include "neuron/model_data.hpp" + +#include + +#include + +#if defined(NRN_ENABLE_GPU) && defined(_OPENACC) +#include +#endif + +using namespace neuron::gpu; + +namespace { +struct DeferDeleteScope { + std::vector ptrs{}; + DeferDeleteScope() { + neuron::container::detail::defer_delete_storage = &ptrs; + } + ~DeferDeleteScope() { + for (void* ptr: ptrs) { + operator delete[](ptr); + } + neuron::container::detail::defer_delete_storage = nullptr; + } +}; +} // namespace + +// Standalone GPU tests do not link container.cpp. +namespace neuron::container::detail { +std::vector* defer_delete_storage{}; +} + +TEST_CASE("upload mirrors sorted node SOA vectors", "[gpu][upload]") { +#if !defined(NRN_ENABLE_GPU) || !defined(_OPENACC) + SKIP("NRN_ENABLE_GPU with OpenACC required"); +#else + if (acc_get_num_devices(acc_device_nvidia) < 1) { + SKIP("No NVIDIA GPU available"); + } + acc_init(acc_device_nvidia); + acc_set_device_num(0, acc_device_nvidia); + + DeferDeleteScope defer_delete{}; + neuron::cache::Model cache{}; + neuron::container::Node::storage nodes{}; + neuron::container::Node::owning_handle n0{nodes}; + neuron::container::Node::owning_handle n1{nodes}; + n0.v() = 10.0; + n1.v() = 20.0; + + auto node_token = nodes.issue_frozen_token(); + neuron::model_sorted_token sorted{cache, std::move(node_token)}; + device_token token{sorted}; + REQUIRE(token.is_on_device()); + REQUIRE(detail::mirror_count_for_testing() > 0); + REQUIRE(detail::is_present_for_testing(&n0.v())); + REQUIRE(detail::is_present_for_testing(&n1.v())); +#endif +} + +TEST_CASE("InterleaveInfo permute-2 upload patches device pointers", "[gpu][upload][interleave]") { +#if !defined(NRN_ENABLE_GPU) || !defined(_OPENACC) + SKIP("NRN_ENABLE_GPU with OpenACC required"); +#else + if (acc_get_num_devices(acc_device_nvidia) < 1) { + SKIP("No NVIDIA GPU available"); + } + acc_init(acc_device_nvidia); + acc_set_device_num(0, acc_device_nvidia); + + neuron::InterleaveInfo info{}; + info.nwarp = 1; + info.nstride = 2; + info.stride = new int[2]{1, 1}; + info.firstnode = new int[2]{0, 1}; + info.lastnode = new int[2]{0, 1}; + info.stridedispl = new int[2]{0, 2}; + info.cellsize = new int[1]{2}; + + UploadState state{}; + detail::upload_interleave_info_for_testing(2, info, 0, state); + REQUIRE(state.mirror_count() >= 6); + REQUIRE(state.is_present(info.stride)); + REQUIRE(state.is_present(info.firstnode)); + REQUIRE(state.is_present(info.stridedispl)); + REQUIRE(state.is_present(&info)); + REQUIRE(nrn_target_deviceptr(&info) != nullptr); + REQUIRE(nrn_target_deviceptr(info.stride) != nullptr); + + state.teardown(); + delete[] info.stride; + delete[] info.firstnode; + delete[] info.lastnode; + delete[] info.stridedispl; + delete[] info.cellsize; + info.stride = nullptr; + info.firstnode = nullptr; + info.lastnode = nullptr; + info.stridedispl = nullptr; + info.cellsize = nullptr; +#endif +} diff --git a/test/unit_tests/gpu/upload_link_stubs.cpp b/test/unit_tests/gpu/upload_link_stubs.cpp new file mode 100644 index 0000000000..f6fd2477ee --- /dev/null +++ b/test/unit_tests/gpu/upload_link_stubs.cpp @@ -0,0 +1,36 @@ +#include "coreneuron/permute/cellorder.hpp" +#include "neuron/container/soa_identifier.hpp" +#include "neuron/gpu/device_state.hpp" +#include "neuron/model_data.hpp" + +// Standalone GPU unit tests link upload.cpp without libnrniv/container.cpp. +namespace neuron { +Model::Model() { + m_node_data.set_unsorted_callback([]() { + cache::model.reset(); + gpu::invalidate_device_state(); + }); +} +Model::~Model() = default; +} // namespace neuron + +namespace neuron::detail { +Model model_data; +} // namespace neuron::detail + +namespace neuron::container::detail { +void notify_handle_dying(non_owning_identifier_without_container /*handle*/) {} +} // namespace neuron::container::detail + +extern int nrn_nthread = 0; + +namespace neuron { +int interleave_permute_type = 0; +InterleaveInfo* interleave_info = nullptr; + +#if defined(NRN_ENABLE_GPU) +int interleave_ncell_for_thread(int /*ith*/) { + return 0; +} +#endif +} // namespace neuron