Skip to content

Commit e6ddbae

Browse files
nrnhinesnrnhines
committed
ninja format-pr
1 parent 88b4366 commit e6ddbae

22 files changed

Lines changed: 228 additions & 166 deletions

‎cmake/NeuronTestHelper.cmake‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -210,7 +210,8 @@ function(nrn_add_test_group)
210210
# Construct the names of the important output files
211211
set(special "${nrnivmodl_directory}/${CMAKE_HOST_SYSTEM_PROCESSOR}/special")
212212
set(nrnmech_lib
213-
"${nrnivmodl_directory}/${CMAKE_HOST_SYSTEM_PROCESSOR}/${CMAKE_SHARED_LIBRARY_PREFIX}nrnmech${CMAKE_SHARED_LIBRARY_SUFFIX}")
213+
"${nrnivmodl_directory}/${CMAKE_HOST_SYSTEM_PROCESSOR}/${CMAKE_SHARED_LIBRARY_PREFIX}nrnmech${CMAKE_SHARED_LIBRARY_SUFFIX}"
214+
)
214215
# Add the custom command to generate the binaries. Get nrnivmodl from the build directory. At
215216
# the moment it seems that `nrnivmodl` is generated at configure time, so there is no target
216217
# to depend on and it should always be available, but it will try and link against libnrniv.so
@@ -219,10 +220,11 @@ function(nrn_add_test_group)
219220
# be a wrapper that invokes CMake?
220221
set(output_binaries "${special}")
221222
list(APPEND nrnivmodl_dependencies nrniv_lib)
222-
# Re-run nrnivmodl when libnrniv.so changes so test special does not retain stale
223-
# NVHPC pgcudafat object paths from a prior libnrniv link line.
224-
list(APPEND nrnivmodl_dependencies
225-
"${CMAKE_BINARY_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}nrniv${CMAKE_SHARED_LIBRARY_SUFFIX}")
223+
# Re-run nrnivmodl when libnrniv.so changes so test special does not retain stale NVHPC
224+
# pgcudafat object paths from a prior libnrniv link line.
225+
list(
226+
APPEND nrnivmodl_dependencies
227+
"${CMAKE_BINARY_DIR}/lib/${CMAKE_SHARED_LIBRARY_PREFIX}nrniv${CMAKE_SHARED_LIBRARY_SUFFIX}")
226228
if(NRN_ENABLE_CORENEURON AND NRN_ADD_TEST_GROUP_CORENEURON)
227229
list(APPEND output_binaries "${special}-core")
228230
if((NOT coreneuron_FOUND) AND (NOT DEFINED CORENEURON_BUILTIN_MODFILES))
@@ -429,8 +431,7 @@ function(nrn_add_test)
429431
list(TRANSFORM test_env REPLACE "^PATH=" "PATH=${_nrn_test_mech_dir}:")
430432
# pytest/python3 launches do not search PATH for dlopen(libnrnmech.so).
431433
if("LD_LIBRARY_PATH" IN_LIST test_env_var_names)
432-
list(TRANSFORM test_env REPLACE "^LD_LIBRARY_PATH="
433-
"LD_LIBRARY_PATH=${_nrn_test_mech_dir}:")
434+
list(TRANSFORM test_env REPLACE "^LD_LIBRARY_PATH=" "LD_LIBRARY_PATH=${_nrn_test_mech_dir}:")
434435
else()
435436
list(APPEND test_env "LD_LIBRARY_PATH=${_nrn_test_mech_dir}")
436437
endif()

‎share/lib/python/neuron/gpu.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -136,4 +136,4 @@ def _sync_to_hoc(self):
136136
pc.gpu_download_flush_interval(int(self._download_flush_interval))
137137

138138

139-
sys.modules[__name__] = gpu()
139+
sys.modules[__name__] = gpu()

‎src/neuron/CMakeLists.txt‎

Lines changed: 20 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,8 @@
22
# NEURON 9.0 model-data layer (src/neuron/) — GPU offload scaffold and future native GPU engine
33
# sources.
44
# =============================================================================
5-
add_library(neuron_gpu STATIC gpu/offload.cpp gpu/device_state.cpp gpu/config.cpp gpu/device_assign.cpp)
5+
add_library(neuron_gpu STATIC gpu/offload.cpp gpu/device_state.cpp gpu/config.cpp
6+
gpu/device_assign.cpp)
67
set_property(TARGET neuron_gpu PROPERTY POSITION_INDEPENDENT_CODE ON)
78
target_include_directories(neuron_gpu PUBLIC ${PROJECT_SOURCE_DIR}/src)
89
target_link_libraries(neuron_gpu PRIVATE fmt::fmt)
@@ -20,9 +21,16 @@ if(NRN_ENABLE_GPU AND CORENRN_ENABLE_GPU)
2021
"CORENEURON_ENABLE_GPU;NRN_ENABLE_GPU")
2122

2223
# upload.cpp pulls NEURON/oc headers; compile as OpenACC object lib for libnrniv + GPU tests.
23-
add_library(neuron_gpu_upload OBJECT gpu/upload.cpp gpu/upload_mechanisms.cpp
24-
gpu/download.cpp gpu/fadvance_gpu.cpp gpu/net_events.cpp
25-
gpu/net_send_buffer.cpp gpu/post_solve.cpp gpu/sync.cpp)
24+
add_library(
25+
neuron_gpu_upload OBJECT
26+
gpu/upload.cpp
27+
gpu/upload_mechanisms.cpp
28+
gpu/download.cpp
29+
gpu/fadvance_gpu.cpp
30+
gpu/net_events.cpp
31+
gpu/net_send_buffer.cpp
32+
gpu/post_solve.cpp
33+
gpu/sync.cpp)
2634
set_property(TARGET neuron_gpu_upload PROPERTY POSITION_INDEPENDENT_CODE ON)
2735
target_include_directories(
2836
neuron_gpu_upload
@@ -36,8 +44,14 @@ if(NRN_ENABLE_GPU AND CORENRN_ENABLE_GPU)
3644
${PROJECT_SOURCE_DIR}/external/fmt/include)
3745
target_link_libraries(neuron_gpu_upload PRIVATE fmt::fmt)
3846
set_source_files_properties(
39-
gpu/upload.cpp gpu/upload_mechanisms.cpp gpu/download.cpp gpu/fadvance_gpu.cpp
40-
gpu/net_events.cpp gpu/net_send_buffer.cpp gpu/post_solve.cpp gpu/sync.cpp
47+
gpu/upload.cpp
48+
gpu/upload_mechanisms.cpp
49+
gpu/download.cpp
50+
gpu/fadvance_gpu.cpp
51+
gpu/net_events.cpp
52+
gpu/net_send_buffer.cpp
53+
gpu/post_solve.cpp
54+
gpu/sync.cpp
4155
PROPERTIES COMPILE_OPTIONS "${_neuron_gpu_acc_flags}" COMPILE_DEFINITIONS
4256
"CORENEURON_ENABLE_GPU;NRN_ENABLE_GPU")
4357
endif()

‎src/neuron/gpu/device_assign.cpp‎

Lines changed: 12 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,11 @@ int mpi_local_rank() {
2929
#if NRNMPI
3030
if (&nrnmpi_use != nullptr && nrnmpi_use != 0) {
3131
MPI_Comm local_comm{};
32-
MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, mpi_simulation_rank(),
33-
MPI_INFO_NULL, &local_comm);
32+
MPI_Comm_split_type(MPI_COMM_WORLD,
33+
MPI_COMM_TYPE_SHARED,
34+
mpi_simulation_rank(),
35+
MPI_INFO_NULL,
36+
&local_comm);
3437
int local_rank = 0;
3538
MPI_Comm_rank(local_comm, &local_rank);
3639
MPI_Comm_free(&local_comm);
@@ -44,8 +47,11 @@ int mpi_local_size() {
4447
#if NRNMPI
4548
if (&nrnmpi_use != nullptr && nrnmpi_use != 0) {
4649
MPI_Comm local_comm{};
47-
MPI_Comm_split_type(MPI_COMM_WORLD, MPI_COMM_TYPE_SHARED, mpi_simulation_rank(),
48-
MPI_INFO_NULL, &local_comm);
50+
MPI_Comm_split_type(MPI_COMM_WORLD,
51+
MPI_COMM_TYPE_SHARED,
52+
mpi_simulation_rank(),
53+
MPI_INFO_NULL,
54+
&local_comm);
4955
int local_size = 1;
5056
MPI_Comm_size(local_comm, &local_size);
5157
MPI_Comm_free(&local_comm);
@@ -87,8 +93,8 @@ void assign_device() {
8793
g_assigned_device_id.store(device_id);
8894

8995
if (mpi_simulation_rank() == 0) {
90-
std::cout << " Info : " << num_devices_per_node << " GPUs shared by "
91-
<< mpi_local_size() << " ranks per node\n";
96+
std::cout << " Info : " << num_devices_per_node << " GPUs shared by " << mpi_local_size()
97+
<< " ranks per node\n";
9298
}
9399
#else
94100
(void) 0;

‎src/neuron/gpu/net_send_buffer.cpp‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -98,14 +98,13 @@ void update_net_send_buffer_on_host(NrnThread* nt, NetSendBuffer_t* nsb) {
9898
return;
9999
}
100100
if (nsb->_cnt > nsb->_size) {
101-
fprintf(stderr,
102-
"ERROR: NetSendBuffer exceeded during GPU execution (thread %d)\n",
103-
nt->id);
101+
fprintf(stderr, "ERROR: NetSendBuffer exceeded during GPU execution (thread %d)\n", nt->id);
104102
std::abort();
105103
}
106104
if (!nsb->_cnt) {
107105
return;
108106
}
107+
// clang-format off
109108
nrn_pragma_acc(update self(nsb->_sendtype[:nsb->_cnt],
110109
nsb->_vdata_index[:nsb->_cnt],
111110
nsb->_pnt_index[:nsb->_cnt],
@@ -120,6 +119,7 @@ void update_net_send_buffer_on_host(NrnThread* nt, NetSendBuffer_t* nsb) {
120119
nsb->_nsb_t[:nsb->_cnt],
121120
nsb->_nsb_flag[:nsb->_cnt])
122121
if (nsb->_cnt))
122+
// clang-format on
123123
#else
124124
(void) nt;
125125
(void) nsb;
@@ -131,12 +131,15 @@ void deliver_net_send_buffer_events(NrnThread* nt, NetSendBuffer_t* nsb) {
131131
return;
132132
}
133133
for (int i = 0; i < nsb->_cnt; ++i) {
134-
auto* const pnt = reinterpret_cast<Point_process*>(static_cast<intptr_t>(nsb->_pnt_index[i]));
134+
auto* const pnt = reinterpret_cast<Point_process*>(
135+
static_cast<intptr_t>(nsb->_pnt_index[i]));
135136
auto* const weight = nsb->_weight_index[i] >= 0
136-
? reinterpret_cast<double*>(static_cast<intptr_t>(nsb->_weight_index[i]))
137+
? reinterpret_cast<double*>(
138+
static_cast<intptr_t>(nsb->_weight_index[i]))
137139
: nullptr;
138140
auto* const vdata = nsb->_vdata_index[i] >= 0
139-
? reinterpret_cast<void*>(static_cast<intptr_t>(nsb->_vdata_index[i]))
141+
? reinterpret_cast<void*>(
142+
static_cast<intptr_t>(nsb->_vdata_index[i]))
140143
: nullptr;
141144
switch (nsb->_sendtype[i]) {
142145
case 0:

‎src/neuron/gpu/post_solve.cpp‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,13 @@ void second_order_cur_on_device(model_sorted_token const& sorted_token, NrnThrea
3838
double* const cur = ml_cache.data_array_ptr<ion_cur_index, 1>();
3939
double* const dcurdv = ml_cache.data_array_ptr<ion_dcurdv_index, 1>();
4040
int* const ni = ml->nodeindices;
41+
// clang-format off
4142
nrn_pragma_acc(parallel loop present(cur [0:count],
4243
dcurdv [0:count],
4344
ni [0:count],
4445
vec_rhs [0:nt.end]) if (nt.compute_gpu)
4546
async(nt.stream_id))
47+
// clang-format on
4648
nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu))
4749
for (int i = 0; i < count; ++i) {
4850
cur[i] += dcurdv[i] * vec_rhs[ni[i]];
@@ -57,15 +59,19 @@ void update_voltage_on_device(NrnThread& nt) {
5759
auto* const vec_rhs = nt.node_rhs_storage();
5860
auto* const vec_v = nt.node_voltage_storage();
5961
if (secondorder) {
62+
// clang-format off
6063
nrn_pragma_acc(parallel loop present(vec_v [0:nt.end], vec_rhs [0:nt.end]) if (nt.compute_gpu)
6164
async(nt.stream_id))
65+
// clang-format on
6266
nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu))
6367
for (int i = 0; i < nt.end; ++i) {
6468
vec_v[i] += 2. * vec_rhs[i];
6569
}
6670
} else {
71+
// clang-format off
6772
nrn_pragma_acc(parallel loop present(vec_v [0:nt.end], vec_rhs [0:nt.end]) if (nt.compute_gpu)
6873
async(nt.stream_id))
74+
// clang-format on
6975
nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu))
7076
for (int i = 0; i < nt.end; ++i) {
7177
vec_v[i] += vec_rhs[i];
@@ -89,11 +95,13 @@ void capacity_current_on_device(model_sorted_token const& sorted_token, NrnThrea
8995
double* const i_cap = ml_cache.data_array_ptr<cap_i_cap_index, 1>();
9096
int* const ni = ml->nodeindices;
9197
double const cfac = .001 * nt.cj;
98+
// clang-format off
9299
nrn_pragma_acc(parallel loop present(vec_rhs [0:nt.end],
93100
cm [0:count],
94101
i_cap [0:count],
95102
ni [0:count]) if (nt.compute_gpu)
96103
async(nt.stream_id))
104+
// clang-format on
97105
nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu))
98106
for (int i = 0; i < count; ++i) {
99107
i_cap[i] = cfac * cm[i] * vec_rhs[ni[i]];
@@ -115,11 +123,13 @@ void fast_imem_on_device(NrnThread& nt) {
115123
if (!vec_sav_d || !vec_sav_rhs) {
116124
return;
117125
}
126+
// clang-format off
118127
nrn_pragma_acc(parallel loop present(vec_rhs [0:nt.end],
119128
vec_area [0:nt.end],
120129
vec_sav_d [0:nt.end],
121130
vec_sav_rhs [0:nt.end]) if (nt.compute_gpu)
122131
async(nt.stream_id))
132+
// clang-format on
123133
nrn_pragma_omp(target teams distribute parallel for simd if(nt.compute_gpu))
124134
for (int i = 0; i < nt.end; ++i) {
125135
vec_sav_rhs[i] = (vec_sav_d[i] * vec_rhs[i] + vec_sav_rhs[i]) * vec_area[i] * 0.01;

‎src/neuron/gpu/sync.cpp‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,8 @@ void sync_matrix_arrays_to_device(NrnThread& nt) {
4646
async(nt.stream_id))
4747
nrn_pragma_omp(target update to(vec_rhs [0:nt.end], vec_d [0:nt.end]) if (nt.compute_gpu))
4848
if (auto* const vec_sav_rhs = nt.node_sav_rhs_storage()) {
49-
nrn_pragma_acc(update device(vec_sav_rhs [0:nt.end]) if (nt.compute_gpu) async(nt.stream_id))
49+
nrn_pragma_acc(update device(vec_sav_rhs [0:nt.end]) if (nt.compute_gpu)
50+
async(nt.stream_id))
5051
nrn_pragma_omp(target update to(vec_sav_rhs [0:nt.end]) if (nt.compute_gpu))
5152
}
5253
if (auto* const vec_sav_d = nt.node_sav_d_storage()) {

‎src/neuron/gpu/upload.cpp‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -209,9 +209,9 @@ void UploadState::teardown() {
209209
nrn_target_delete(const_cast<Memb_list**>(static_cast<Memb_list* const*>(mirror.host)),
210210
mirror.count);
211211
} else if (mirror.sizeof_elem == sizeof(NetSendBuffer_t)) {
212-
nrn_target_delete(
213-
const_cast<NetSendBuffer_t*>(static_cast<NetSendBuffer_t const*>(mirror.host)),
214-
mirror.count);
212+
nrn_target_delete(const_cast<NetSendBuffer_t*>(
213+
static_cast<NetSendBuffer_t const*>(mirror.host)),
214+
mirror.count);
215215
}
216216
}
217217
#endif

‎src/neuron/gpu/upload_mechanisms.cpp‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,10 @@ int mechanism_padded_count(int count) {
2424
return ((count + k_soa_pad - 1) / k_soa_pad) * k_soa_pad;
2525
}
2626

27-
void record_upload(UploadState& state, void const* host, std::size_t count, std::size_t sizeof_elem) {
27+
void record_upload(UploadState& state,
28+
void const* host,
29+
std::size_t count,
30+
std::size_t sizeof_elem) {
2831
state.record(host, count, sizeof_elem);
2932
}
3033

@@ -82,8 +85,7 @@ void upload_mechanism_pdata(Memb_list* ml, int type, Memb_list* d_ml, UploadStat
8285
device_row_ptrs[static_cast<std::size_t>(i)] = d_row;
8386
}
8487

85-
Datum** const d_pdata_rows =
86-
nrn_target_copyin(device_row_ptrs.data(), device_row_ptrs.size());
88+
Datum** const d_pdata_rows = nrn_target_copyin(device_row_ptrs.data(), device_row_ptrs.size());
8789
record_upload(state, device_row_ptrs.data(), device_row_ptrs.size(), sizeof(Datum*));
8890
nrn_target_memcpy_to_device(&(d_ml->pdata), &d_pdata_rows, 1);
8991
}
@@ -104,7 +106,8 @@ void upload_mechanism_shell(Memb_list* ml, int type, UploadState& state) {
104106

105107
int const thread_size = memb_func[type].thread_size_;
106108
if (thread_size > 0 && ml->_thread) {
107-
Datum* const d_thread = nrn_target_copyin(ml->_thread, static_cast<std::size_t>(thread_size));
109+
Datum* const d_thread = nrn_target_copyin(ml->_thread,
110+
static_cast<std::size_t>(thread_size));
108111
record_upload(state, ml->_thread, static_cast<std::size_t>(thread_size), sizeof(Datum));
109112
nrn_target_memcpy_to_device(&(d_ml->_thread), &d_thread, 1);
110113
}
@@ -124,8 +127,7 @@ void upload_thread_ml_list(NrnThread& nt, UploadState& state) {
124127
std::vector<Memb_list*> device_ptrs(static_cast<std::size_t>(n_type), nullptr);
125128
for (int type = 0; type < n_type; ++type) {
126129
if (nt._ml_list[type]) {
127-
device_ptrs[static_cast<std::size_t>(type)] =
128-
nrn_target_deviceptr(nt._ml_list[type]);
130+
device_ptrs[static_cast<std::size_t>(type)] = nrn_target_deviceptr(nt._ml_list[type]);
129131
}
130132
}
131133
nrn_target_memcpy_to_device(d_ml_list, device_ptrs.data(), static_cast<std::size_t>(n_type));

‎src/nmodl/codegen/codegen_neuron_acc_visitor.cpp‎

Lines changed: 18 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -177,13 +177,15 @@ void CodegenNeuronAccVisitor::print_net_send_call(const ast::FunctionCall& node)
177177
} else {
178178
const auto& t = get_variable_name("t");
179179
printer->add_text("net_send_buffering(");
180-
std::string weight_ptr = weight_index == "0" ? "0" : fmt::format("(intptr_t){}", weight_index);
181-
printer->fmt_text("nt, ml, ml->_net_send_buffer, 0, (intptr_t)&{}, {}, "
182-
"(intptr_t){}, {}+",
183-
tqitem,
184-
weight_ptr,
185-
point_process,
186-
t);
180+
std::string weight_ptr = weight_index == "0" ? "0"
181+
: fmt::format("(intptr_t){}", weight_index);
182+
printer->fmt_text(
183+
"nt, ml, ml->_net_send_buffer, 0, (intptr_t)&{}, {}, "
184+
"(intptr_t){}, {}+",
185+
tqitem,
186+
weight_ptr,
187+
point_process,
188+
t);
187189
}
188190
print_vector_elements(arguments, ", ");
189191
printer->add_text(')');
@@ -203,10 +205,11 @@ void CodegenNeuronAccVisitor::print_net_move_call(const ast::FunctionCall& node)
203205
return;
204206
}
205207
printer->add_text("net_send_buffering(");
206-
printer->fmt_text("nt, ml, ml->_net_send_buffer, 2, (intptr_t)&{}, (intptr_t)-1, "
207-
"(intptr_t){}, ",
208-
tqitem,
209-
point_process);
208+
printer->fmt_text(
209+
"nt, ml, ml->_net_send_buffer, 2, (intptr_t)&{}, (intptr_t)-1, "
210+
"(intptr_t){}, ",
211+
tqitem,
212+
point_process);
210213
print_vector_elements(node.get_arguments(), ", ");
211214
printer->add_text(", 0.0, 0.0");
212215
printer->add_text(")");
@@ -222,9 +225,10 @@ void CodegenNeuronAccVisitor::print_net_event_call(const ast::FunctionCall& node
222225
}
223226
const auto& point_process = get_variable_name(naming::POINT_PROCESS_VARIABLE, false);
224227
printer->add_text("net_send_buffering(");
225-
printer->fmt_text("nt, ml, ml->_net_send_buffer, 1, (intptr_t)-1, (intptr_t)-1, "
226-
"(intptr_t){}, ",
227-
point_process);
228+
printer->fmt_text(
229+
"nt, ml, ml->_net_send_buffer, 1, (intptr_t)-1, (intptr_t)-1, "
230+
"(intptr_t){}, ",
231+
point_process);
228232
print_vector_elements(arguments, ", ");
229233
printer->add_text(", 0.0, 0.0");
230234
printer->add_text(")");

0 commit comments

Comments
 (0)