Skip to content

Commit d2a76ab

Browse files
iraikovalexsavulescunrnhines
authored
Implementation of subworlds in CoreNEURON (#2185)
 Co-authored-by: Alexandru Săvulescu <alexandru.savulescu@epfl.ch> Co-authored-by: nrnhines <michael.hines@yale.edu>
1 parent 3adaa7d commit d2a76ab

12 files changed

Lines changed: 230 additions & 16 deletions

File tree

‎.pre-commit-config.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
repos:
22
- repo: https://github.com/psf/black
3-
rev: 22.1.0
3+
rev: 22.12.0
44
hooks:
55
- id: black
66
language_version: python3

‎src/coreneuron/mpi/lib/nrnmpi.cpp‎

Lines changed: 55 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include <mpi.h>
2121
namespace coreneuron {
2222

23+
2324
MPI_Comm nrnmpi_world_comm;
2425
MPI_Comm nrnmpi_comm;
2526
int nrnmpi_numprocs_;
@@ -34,6 +35,8 @@ static void nrn_fatal_error(const char* msg) {
3435
nrnmpi_abort_impl(-1);
3536
}
3637

38+
void corenrn_subworld();
39+
3740
nrnmpi_init_ret_t nrnmpi_init_impl(int* pargc, char*** pargv, bool is_quiet) {
3841
// Execute at most once per launch. Avoid memory leak.
3942
static bool executed = false;
@@ -54,10 +57,13 @@ nrnmpi_init_ret_t nrnmpi_init_impl(int* pargc, char*** pargv, bool is_quiet) {
5457
nrn_assert(MPI_Init(pargc, pargv) == MPI_SUCCESS);
5558
#endif
5659
}
60+
5761
nrn_assert(MPI_Comm_dup(MPI_COMM_WORLD, &nrnmpi_world_comm) == MPI_SUCCESS);
5862
nrn_assert(MPI_Comm_dup(nrnmpi_world_comm, &nrnmpi_comm) == MPI_SUCCESS);
59-
nrn_assert(MPI_Comm_rank(nrnmpi_world_comm, &nrnmpi_myid_) == MPI_SUCCESS);
60-
nrn_assert(MPI_Comm_size(nrnmpi_world_comm, &nrnmpi_numprocs_) == MPI_SUCCESS);
63+
corenrn_subworld(); // split nrnmpi_comm if ParallelContext.subworlds has been used
64+
nrn_assert(MPI_Comm_rank(nrnmpi_comm, &nrnmpi_myid_) == MPI_SUCCESS);
65+
nrn_assert(MPI_Comm_size(nrnmpi_comm, &nrnmpi_numprocs_) == MPI_SUCCESS);
66+
6167
nrnmpi_spike_initialize();
6268

6369
if (nrnmpi_myid_ == 0 && !is_quiet) {
@@ -82,6 +88,53 @@ void nrnmpi_finalize_impl(void) {
8288
}
8389
}
8490

91+
extern "C" {
92+
extern void (*nrn2core_subworld_info_)(int&, int&, int&, int&, int&);
93+
}
94+
95+
void corenrn_subworld() {
96+
// If ParallelContext.subworlds has been invoked, split the world
97+
// communicator according to the subworld partitioning.
98+
static int change_cnt{0};
99+
int nrn_subworld_change_cnt, nrn_subworld_index, nrn_subworld_rank, nrn_mpi_numprocs_subworld,
100+
nrn_mpi_numprocs_world;
101+
if (!nrn2core_subworld_info_) {
102+
return;
103+
}
104+
(*nrn2core_subworld_info_)(nrn_subworld_change_cnt,
105+
nrn_subworld_index,
106+
nrn_subworld_rank,
107+
nrn_mpi_numprocs_subworld,
108+
nrn_mpi_numprocs_world);
109+
if (nrn_subworld_change_cnt == change_cnt) {
110+
return;
111+
}
112+
change_cnt = nrn_subworld_change_cnt;
113+
114+
// clean up / free old nrn_mpi_comm
115+
nrn_assert(MPI_Comm_free(&nrnmpi_comm) == MPI_SUCCESS);
116+
117+
// ensure world size is the same as NEURON
118+
int world_size{-1};
119+
nrn_assert(MPI_Comm_size(nrnmpi_world_comm, &world_size) == MPI_SUCCESS);
120+
nrn_assert(world_size == nrn_mpi_numprocs_world);
121+
122+
// create a new nrnmpi_comm based on subworld partitioning
123+
nrn_assert(
124+
MPI_Comm_split(nrnmpi_world_comm, nrn_subworld_index, nrn_subworld_rank, &nrnmpi_comm) ==
125+
MPI_SUCCESS);
126+
127+
// assert that rank order and size is consistent between NEURON and CoreNEURON
128+
int subworld_rank{-1};
129+
nrn_assert(MPI_Comm_rank(nrnmpi_comm, &subworld_rank) == MPI_SUCCESS);
130+
nrn_assert(nrn_subworld_rank == subworld_rank);
131+
132+
int subworld_size{-1};
133+
nrn_assert(MPI_Comm_size(nrnmpi_comm, &subworld_size) == MPI_SUCCESS);
134+
nrn_assert(subworld_size == nrn_mpi_numprocs_subworld);
135+
}
136+
137+
85138
// check if appropriate threading level supported (i.e. MPI_THREAD_FUNNELED)
86139
void nrnmpi_check_threading_support_impl() {
87140
int th = 0;

‎src/coreneuron/utils/utils.cpp‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,10 @@ double nrn_wtime() {
3131
return (time1.tv_sec + time1.tv_usec / 1.e6);
3232
}
3333
}
34+
35+
extern "C" {
36+
void (*nrn2core_subworld_info_)(int&, int&, int&);
37+
}
38+
39+
3440
} // namespace coreneuron

‎src/nrniv/nrncore_write/callbacks/nrncore_callbacks.cpp‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1221,3 +1221,17 @@ void nrn2core_patternstim(void** info) {
12211221
assert(ml.nodecount == 1);
12221222
*info = nrn_patternstim_info_ref(ml.pdata[0]);
12231223
}
1224+
1225+
1226+
// Info from NEURON subworlds at beginning of psolve.
1227+
void nrn2core_subworld_info(int& cnt,
1228+
int& subworld_index,
1229+
int& subworld_rank,
1230+
int& numprocs_subworld,
1231+
int& numprocs_world) {
1232+
cnt = nrnmpi_subworld_change_cnt;
1233+
subworld_index = nrnmpi_subworld_id;
1234+
subworld_rank = nrnmpi_myid;
1235+
numprocs_subworld = nrnmpi_numprocs_subworld;
1236+
numprocs_world = nrnmpi_numprocs_world;
1237+
}

‎src/nrniv/nrncore_write/callbacks/nrncore_callbacks.h‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -183,6 +183,13 @@ void nrn2core_PreSyn_flag(int tid, std::set<int>& presyns_flag_true);
183183
// Direct transfer with respect to PatternStim
184184
void nrn2core_patternstim(void** info);
185185

186+
// Info from NEURON subworlds at beginning of psolve.
187+
void nrn2core_subworld_info(int& cnt,
188+
int& subworld_index,
189+
int& subworld_rank,
190+
int& subworld_size,
191+
int& numprocs_world);
192+
186193
} // end of extern "C"
187194

188195
static core2nrn_callback_t cnbs[] = {
@@ -227,6 +234,8 @@ static core2nrn_callback_t cnbs[] = {
227234

228235
{"nrn2core_patternstim_", (CNB) nrn2core_patternstim},
229236

237+
{"nrn2core_subworld_info_", (CNB) nrn2core_subworld_info},
238+
230239
{NULL, NULL}};
231240

232241

‎src/nrnmpi/nrnmpi.cpp‎

Lines changed: 15 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,9 @@ for (i=0; i < *pargc; ++i) {
145145
nrnmpi_myid = nrnmpi_myid_bbs = nrnmpi_myid_world;
146146
nrnmpi_spike_initialize();
147147
nrnmpi_use = 1;
148+
nrnmpi_subworld_change_cnt = 0; // increment from within void nrnmpi_subworld_size(int n)
149+
nrnmpi_subworld_id = 0; // Subworld index of current rank
150+
nrnmpi_numprocs_subworld = nrnmpi_numprocs_bbs; // Size of subworld of current rank
148151

149152
/*begin instrumentation*/
150153
#if USE_HPM
@@ -216,6 +219,8 @@ void nrnmpi_abort(int errcode) {
216219
}
217220

218221
#if NRNMPI
222+
223+
219224
void nrnmpi_subworld_size(int n) {
220225
/* n is the (desired) size of a subworld (pc.nhost) */
221226
/* A subworld (net) is contiguous */
@@ -254,6 +259,8 @@ void nrnmpi_subworld_size(int n) {
254259
asrt(MPI_Comm_size(nrnmpi_comm, &nrnmpi_numprocs));
255260
asrt(MPI_Comm_rank(nrn_bbs_comm, &nrnmpi_myid_bbs));
256261
asrt(MPI_Comm_size(nrn_bbs_comm, &nrnmpi_numprocs_bbs));
262+
nrnmpi_subworld_id = nrnmpi_myid_bbs;
263+
nrnmpi_numprocs_subworld = nrnmpi_numprocs_bbs;
257264
} else if (n == nrnmpi_numprocs_world) {
258265
asrt(MPI_Group_incl(wg, 1, &r, &grp_bbs));
259266
asrt(MPI_Comm_dup(nrnmpi_world_comm, &nrnmpi_comm));
@@ -267,6 +274,8 @@ void nrnmpi_subworld_size(int n) {
267274
nrnmpi_myid_bbs = -1;
268275
nrnmpi_numprocs_bbs = -1;
269276
}
277+
nrnmpi_subworld_id = 0;
278+
nrnmpi_numprocs_subworld = nrnmpi_numprocs;
270279
} else {
271280
int nw = nrnmpi_numprocs_world;
272281
int nb = nw / n; /* nrnmpi_numprocs_bbs */
@@ -300,15 +309,16 @@ void nrnmpi_subworld_size(int n) {
300309
asrt(MPI_Comm_rank(nrn_bbs_comm, &nrnmpi_myid_bbs));
301310
asrt(MPI_Comm_size(nrn_bbs_comm, &nrnmpi_numprocs_bbs));
302311
} else {
303-
#if 1
304312
nrnmpi_myid_bbs = -1;
305313
nrnmpi_numprocs_bbs = -1;
306-
#else
307-
nrnmpi_myid_bbs = r / n;
308-
nrnmpi_numprocs_bbs = nb;
309-
#endif
314+
}
315+
nrnmpi_subworld_id = r / n;
316+
nrnmpi_numprocs_subworld = n;
317+
if ((nw % n != 0) && (nrnmpi_subworld_id == (nb - 1))) {
318+
nrnmpi_numprocs_subworld = nw % n; /* and the last will have pc.nhost = nw%n */
310319
}
311320
}
321+
nrnmpi_subworld_change_cnt++;
312322
asrt(MPI_Group_free(&wg));
313323
}
314324

‎src/nrnmpi/nrnmpi_def_cinc‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ int nrnmpi_numprocs_world = 1;
55
int nrnmpi_myid_world = 0;
66
int nrnmpi_numprocs_bbs = 1;
77
int nrnmpi_myid_bbs = 0;
8+
// increment from within void nrnmpi_subworld_size(int n)
9+
int nrnmpi_subworld_change_cnt = 0;
10+
int nrnmpi_subworld_id = -1;
11+
int nrnmpi_numprocs_subworld = 1;
812

913
int nrnmpi_nout_;
1014
int* nrnmpi_nin_;

‎src/oc/nrnmpi.h‎

Lines changed: 9 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,15 @@
77
not easily coexist. ParallelContext.subworlds(nsmall) divides the world into
88
nrnmpi_numprocs_world/small subworlds of size nsmall.
99
*/
10-
extern int nrnmpi_numprocs_world; /* size of entire world. total size of all subworlds */
11-
extern int nrnmpi_myid_world; /* rank in entire world */
12-
extern int nrnmpi_numprocs; /* size of subworld */
13-
extern int nrnmpi_myid; /* rank in subworld */
14-
extern int nrnmpi_numprocs_bbs; /* number of subworlds */
15-
extern int nrnmpi_myid_bbs; /* rank in nrn_bbs_comm of rank 0 of a subworld */
10+
extern int nrnmpi_numprocs_world; /* size of entire world. total size of all subworlds */
11+
extern int nrnmpi_myid_world; /* rank in entire world */
12+
extern int nrnmpi_numprocs; /* size of subworld */
13+
extern int nrnmpi_myid; /* rank in subworld */
14+
extern int nrnmpi_numprocs_bbs; /* number of subworlds */
15+
extern int nrnmpi_myid_bbs; /* rank in nrn_bbs_comm of rank 0 of a subworld */
16+
extern int nrnmpi_subworld_change_cnt; /* increment from within void nrnmpi_subworld_size(int n) */
17+
extern int nrnmpi_subworld_id; /* subworld index on all ranks */
18+
extern int nrnmpi_numprocs_subworld; /* number of ranks in subworld on all ranks */
1619

1720
typedef struct {
1821
int gid;

‎src/parallel/bbs.cpp‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -56,7 +56,6 @@ void BBS::init(int) {
5656
if (!BBSImpl::started_) {
5757
BBSImpl::is_master_ = (nrnmpi_myid_bbs == 0) ? true : false;
5858
BBSImpl::master_works_ = true;
59-
// printf("%d BBS::init is_master=%d\n", nrnmpi_myid_bbs, BBSImpl::is_master_);
6059
}
6160
// Just as with PVM which stored buffers on the bulletin board
6261
// so we have the following files to store MPI_PACKED buffers
@@ -427,6 +426,12 @@ void BBSImpl::worker() {
427426
// forever request and execute commands
428427
double st, et;
429428
int id;
429+
if (debug) {
430+
printf("%d BBS::worker is_master=%d nrnmpi_myid = %d\n",
431+
nrnmpi_myid_world,
432+
is_master(),
433+
nrnmpi_myid);
434+
}
430435
if (!is_master()) {
431436
if (nrnmpi_myid_bbs == -1) { // wait for message from
432437
for (;;) { // the proper nrnmpi_myid == 0

‎test/CMakeLists.txt‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -346,6 +346,16 @@ if(NRN_ENABLE_PYTHON AND PYTEST_FOUND)
346346
${MPIEXEC_POSTFLAGS}
347347
-notatty
348348
-python)
349+
set(modtests_launch_py_mpi_subworlds
350+
${MPIEXEC_NAME}
351+
${MPIEXEC_NUMPROC_FLAG}
352+
6
353+
${MPIEXEC_OVERSUBSCRIBE}
354+
${MPIEXEC_PREFLAGS}
355+
special
356+
${MPIEXEC_POSTFLAGS}
357+
-notatty
358+
-python)
349359
else()
350360
set(modtests_preload_sanitizer PRELOAD_SANITIZER)
351361
set(modtests_launch_py ${PYTHON_EXECUTABLE} ${pytest})
@@ -359,6 +369,15 @@ if(NRN_ENABLE_PYTHON AND PYTEST_FOUND)
359369
${preload_sanitizer_mpiexec}
360370
${PYTHON_EXECUTABLE}
361371
${MPIEXEC_POSTFLAGS})
372+
set(modtests_launch_py_mpi_subworlds
373+
${MPIEXEC_NAME}
374+
${MPIEXEC_NUMPROC_FLAG}
375+
6
376+
${MPIEXEC_OVERSUBSCRIBE}
377+
${MPIEXEC_PREFLAGS}
378+
${preload_sanitizer_mpiexec}
379+
${PYTHON_EXECUTABLE}
380+
${MPIEXEC_POSTFLAGS})
362381
endif()
363382

364383
# External coreneuron can be used for testing but for simplicity we are testing only submodule
@@ -590,6 +609,15 @@ if(NRN_ENABLE_PYTHON AND PYTEST_FOUND)
590609
${MPIEXEC_NAME} ${MPIEXEC_NUMPROC_FLAG} 2 ${MPIEXEC_OVERSUBSCRIBE} ${MPIEXEC_PREFLAGS}
591610
special ${MPIEXEC_POSTFLAGS} -mpi -python
592611
${PROJECT_SOURCE_DIR}/test/coreneuron/test_inputpresyn.py)
612+
nrn_add_test(
613+
GROUP coreneuron_modtests
614+
NAME test_subworlds_py_${processor}
615+
REQUIRES coreneuron ${processor} ${modtests_preload_sanitizer}
616+
SCRIPT_PATTERNS test/coreneuron/test_subworlds.py
617+
PROCESSORS 6
618+
ENVIRONMENT ${modtests_processor_env} ${nrnpython_mpi_env}
619+
COVERAGE_FILE=.coverage.coreneuron_test_subworlds_py
620+
COMMAND ${modtests_launch_py_mpi_subworlds} test/coreneuron/test_subworlds.py)
593621
endif()
594622
nrn_add_test_group(
595623
NAME nmodl_tests_coreneuron

0 commit comments

Comments
 (0)