Skip to content

Commit 398b8d4

Browse files
mshanthagitmshanthagit
authored
Merge pull request #12985 from amd-nithyavs/17Dec2024_bug_fixes
coll/acoll: A few miscellaneous bugfixes
2 parents b09a79d + 3e67dd1 commit 398b8d4

9 files changed

+294
-167
lines changed

‎ompi/mca/coll/acoll/coll_acoll.h

+7-1
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222

2323
#ifdef HAVE_XPMEM_H
2424
#include "opal/mca/rcache/base/base.h"
25+
#include "opal/class/opal_hash_table.h"
2526
#include <xpmem.h>
2627
#endif
2728

@@ -37,12 +38,14 @@ extern int mca_coll_acoll_max_comms;
3738
extern int mca_coll_acoll_sg_size;
3839
extern int mca_coll_acoll_sg_scale;
3940
extern int mca_coll_acoll_node_size;
41+
extern int mca_coll_acoll_force_numa;
4042
extern int mca_coll_acoll_use_dynamic_rules;
4143
extern int mca_coll_acoll_mnode_enable;
4244
extern int mca_coll_acoll_bcast_lin0;
4345
extern int mca_coll_acoll_bcast_lin1;
4446
extern int mca_coll_acoll_bcast_lin2;
4547
extern int mca_coll_acoll_bcast_nonsg;
48+
extern int mca_coll_acoll_bcast_socket;
4649
extern int mca_coll_acoll_allgather_lin;
4750
extern int mca_coll_acoll_allgather_ring_1;
4851

@@ -123,6 +126,7 @@ typedef struct coll_acoll_data {
123126
void **xpmem_raddr;
124127
mca_rcache_base_module_t **rcache;
125128
void *scratch;
129+
opal_hash_table_t **xpmem_reg_tracker_ht;
126130
#endif
127131
opal_shmem_ds_t *allshmseg_id;
128132
void **allshmmmap_sbuf;
@@ -160,7 +164,7 @@ typedef struct coll_acoll_subcomms {
160164
int numa_root;
161165
int socket_ldr_root;
162166
int base_root[MCA_COLL_ACOLL_NUM_BASE_LYRS][MCA_COLL_ACOLL_NUM_LAYERS];
163-
int base_rank[MCA_COLL_ACOLL_NUM_BASE_LYRS];
167+
int base_rank[MCA_COLL_ACOLL_NUM_BASE_LYRS][MCA_COLL_ACOLL_NUM_LAYERS];
164168
int socket_rank;
165169
int subgrp_size;
166170
int initialized;
@@ -198,12 +202,14 @@ struct mca_coll_acoll_module_t {
198202
int log2_sg_cnt;
199203
int node_cnt;
200204
int log2_node_cnt;
205+
int force_numa;
201206
int use_dyn_rules;
202207
// Todo: Use substructure for every API related ones
203208
int use_mnode;
204209
int use_lin0;
205210
int use_lin1;
206211
int use_lin2;
212+
int use_socket;
207213
int mnode_sg_size;
208214
int mnode_log2_sg_size;
209215
int allg_lin;

‎ompi/mca/coll/acoll/coll_acoll_allgather.c

+6-6
Original file line numberDiff line numberDiff line change
@@ -344,7 +344,7 @@ static inline int mca_coll_acoll_allgather_intra(const void *sbuf, size_t scount
344344
}
345345

346346
/* Return if all ranks belong to single subgroup */
347-
if (num_sgs == 1) {
347+
if (1 == num_sgs) {
348348
/* All done */
349349
return err;
350350
}
@@ -396,7 +396,7 @@ static inline int mca_coll_acoll_allgather_intra(const void *sbuf, size_t scount
396396
}
397397
/* Now all base ranks have the full data */
398398
/* Do broadcast within subgroups from the base ranks for the extra data */
399-
if (sg_id == 0) {
399+
if (0 == sg_id) {
400400
num_data_blks = 1;
401401
data_blk_size[0] = bcount * (num_sgs - 2) + last_subgrp_rcnt;
402402
blk_ofst[0] = bcount;
@@ -527,7 +527,7 @@ int mca_coll_acoll_allgather(const void *sbuf, size_t scount, struct ompi_dataty
527527
if (num_nodes > 1) {
528528
assert(subc->local_r_comm != NULL);
529529
}
530-
intra_comm = num_nodes == 1 ? comm : subc->local_r_comm;
530+
intra_comm = 1 == num_nodes ? comm : subc->local_r_comm;
531531
}
532532
err = mca_coll_acoll_allgather_intra(sbuf, scount, sdtype, local_rbuf, rcount, rdtype,
533533
intra_comm, module);
@@ -536,7 +536,7 @@ int mca_coll_acoll_allgather(const void *sbuf, size_t scount, struct ompi_dataty
536536
}
537537

538538
/* Return if intra-node communicator */
539-
if ((num_nodes == 1) || (size <= 2)) {
539+
if ((1 == num_nodes) || (size <= 2)) {
540540
/* All done */
541541
return err;
542542
}
@@ -592,7 +592,7 @@ int mca_coll_acoll_allgather(const void *sbuf, size_t scount, struct ompi_dataty
592592
} /* End of if inter leader */
593593

594594
/* Do intra node broadcast */
595-
if (node_id == 0) {
595+
if (0 == node_id) {
596596
num_data_blks = 1;
597597
data_blk_size[0] = bcount * (num_nodes - 2) + last_subgrp_rcnt;
598598
blk_ofst[0] = bcount;
@@ -613,7 +613,7 @@ int mca_coll_acoll_allgather(const void *sbuf, size_t scount, struct ompi_dataty
613613
/* Loop over data blocks */
614614
for (i = 0; i < num_data_blks; i++) {
615615
char *buff = (char *) rbuf + (ptrdiff_t) blk_ofst[i] * rext;
616-
err = (comm)->c_coll->coll_bcast(buff, data_blk_size[i], rdtype, 0, subc->local_r_comm,
616+
err = ompi_coll_base_bcast_intra_basic_linear(buff, data_blk_size[i], rdtype, 0, subc->local_r_comm,
617617
module);
618618
if (MPI_SUCCESS != err) {
619619
return err;

‎ompi/mca/coll/acoll/coll_acoll_allreduce.c

+16-17
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ static inline int mca_coll_acoll_reduce_xpmem_h(const void *sbuf, void *rbuf, si
5959
int size;
6060
size_t total_dsize, dsize;
6161

62-
coll_acoll_init(module, comm, subc->data, subc);
62+
coll_acoll_init(module, comm, subc->data, subc, 0);
6363
coll_acoll_data_t *data = subc->data;
6464
if (NULL == data) {
6565
return -1;
@@ -82,15 +82,15 @@ static inline int mca_coll_acoll_reduce_xpmem_h(const void *sbuf, void *rbuf, si
8282
if (!subc->xpmem_use_sr_buf) {
8383
tmp_rbuf = (char *) data->scratch;
8484
tmp_sbuf = (char *) data->scratch + (subc->xpmem_buf_size) / 2;
85-
if ((sbuf == MPI_IN_PLACE)) {
85+
if ((MPI_IN_PLACE == sbuf)) {
8686
memcpy(tmp_sbuf, rbuf, total_dsize);
8787
} else {
8888
memcpy(tmp_sbuf, sbuf, total_dsize);
8989
}
9090
} else {
9191
tmp_sbuf = (char *) sbuf;
9292
tmp_rbuf = (char *) rbuf;
93-
if (sbuf == MPI_IN_PLACE) {
93+
if (MPI_IN_PLACE == sbuf) {
9494
tmp_sbuf = (char *) rbuf;
9595
}
9696
}
@@ -153,7 +153,7 @@ static inline int mca_coll_acoll_reduce_xpmem_h(const void *sbuf, void *rbuf, si
153153

154154
my_count_size = (l2_local_rank == (local_size - 1)) ? chunk + (count % local_size) : chunk;
155155

156-
if (l2_local_rank == 0) {
156+
if (0 == l2_local_rank) {
157157
for (int i = 1; i < local_size; i++) {
158158
ompi_op_reduce(op, (char *) data->xpmem_raddr[l2_gp[i]], (char *) tmp_rbuf,
159159
my_count_size, dtype);
@@ -192,7 +192,7 @@ static inline int mca_coll_acoll_allreduce_xpmem_f(const void *sbuf, void *rbuf,
192192
int size;
193193
size_t total_dsize, dsize;
194194

195-
coll_acoll_init(module, comm, subc->data, subc);
195+
coll_acoll_init(module, comm, subc->data, subc, 0);
196196
coll_acoll_data_t *data = subc->data;
197197
if (NULL == data) {
198198
return -1;
@@ -207,15 +207,15 @@ static inline int mca_coll_acoll_allreduce_xpmem_f(const void *sbuf, void *rbuf,
207207
if (!subc->xpmem_use_sr_buf) {
208208
tmp_rbuf = (char *) data->scratch;
209209
tmp_sbuf = (char *) data->scratch + (subc->xpmem_buf_size) / 2;
210-
if ((sbuf == MPI_IN_PLACE)) {
210+
if ((MPI_IN_PLACE == sbuf)) {
211211
memcpy(tmp_sbuf, rbuf, total_dsize);
212212
} else {
213213
memcpy(tmp_sbuf, sbuf, total_dsize);
214214
}
215215
} else {
216216
tmp_sbuf = (char *) sbuf;
217217
tmp_rbuf = (char *) rbuf;
218-
if (sbuf == MPI_IN_PLACE) {
218+
if (MPI_IN_PLACE == sbuf) {
219219
tmp_sbuf = (char *) rbuf;
220220
}
221221
}
@@ -242,7 +242,7 @@ static inline int mca_coll_acoll_allreduce_xpmem_f(const void *sbuf, void *rbuf,
242242

243243
size_t chunk = count / size;
244244
size_t my_count_size = (rank == (size - 1)) ? (count / size) + count % size : count / size;
245-
if (rank == 0) {
245+
if (0 == rank) {
246246
if (sbuf != MPI_IN_PLACE)
247247
memcpy(tmp_rbuf, sbuf, my_count_size * dsize);
248248
} else {
@@ -299,7 +299,7 @@ void mca_coll_acoll_sync(coll_acoll_data_t *data, int offset, int *group, int gp
299299
opal_atomic_wmb();
300300

301301
int val;
302-
if (up == 1) {
302+
if (1 == up) {
303303
val = data->sync[0];
304304
} else {
305305
val = data->sync[1];
@@ -346,7 +346,7 @@ void mca_coll_acoll_sync(coll_acoll_data_t *data, int offset, int *group, int gp
346346
__ATOMIC_RELAXED);
347347
}
348348
}
349-
if (up == 1) {
349+
if (1 == up) {
350350
data->sync[0] = val;
351351
} else {
352352
data->sync[1] = val;
@@ -361,8 +361,7 @@ int mca_coll_acoll_allreduce_small_msgs_h(const void *sbuf, void *rbuf, size_t c
361361
{
362362
size_t dsize;
363363
int err = MPI_SUCCESS;
364-
365-
coll_acoll_init(module, comm, subc->data, subc);
364+
coll_acoll_init(module, comm, subc->data, subc, 0);
366365
coll_acoll_data_t *data = subc->data;
367366
if (NULL == data) {
368367
return -1;
@@ -434,7 +433,7 @@ int mca_coll_acoll_allreduce_small_msgs_h(const void *sbuf, void *rbuf, size_t c
434433
}
435434

436435
if (intra && (ompi_comm_size(subc->numa_comm) > 1)) {
437-
err = mca_coll_acoll_bcast(rbuf, count, dtype, 0, subc->numa_comm, module);
436+
err = ompi_coll_base_bcast_intra_basic_linear(rbuf, count, dtype, 0, subc->numa_comm, module);
438437
}
439438
return err;
440439
}
@@ -451,7 +450,7 @@ int mca_coll_acoll_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
451450
ompi_datatype_type_size(dtype, &dsize);
452451
total_dsize = dsize * count;
453452

454-
if (size == 1) {
453+
if (1 == size) {
455454
if (MPI_IN_PLACE != sbuf) {
456455
memcpy((char *) rbuf, sbuf, total_dsize);
457456
}
@@ -483,7 +482,7 @@ int mca_coll_acoll_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
483482

484483
alg = coll_allreduce_decision_fixed(size, total_dsize);
485484

486-
if (num_nodes == 1) {
485+
if (1 == num_nodes) {
487486
if (total_dsize < 32) {
488487
return ompi_coll_base_allreduce_intra_recursivedoubling(sbuf, rbuf, count, dtype, op,
489488
comm, module);
@@ -494,10 +493,10 @@ int mca_coll_acoll_allreduce_intra(const void *sbuf, void *rbuf, size_t count,
494493
return ompi_coll_base_allreduce_intra_recursivedoubling(sbuf, rbuf, count, dtype, op,
495494
comm, module);
496495
} else if (total_dsize < 65536) {
497-
if (alg == 1) {
496+
if (1 == alg) {
498497
return ompi_coll_base_allreduce_intra_recursivedoubling(sbuf, rbuf, count, dtype,
499498
op, comm, module);
500-
} else if (alg == 2) {
499+
} else if (2 == alg) {
501500
return ompi_coll_base_allreduce_intra_redscat_allgather(sbuf, rbuf, count, dtype,
502501
op, comm, module);
503502
} else { /*alg == 3 */

‎ompi/mca/coll/acoll/coll_acoll_barrier.c

+1-1
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,7 @@ int mca_coll_acoll_barrier_intra(struct ompi_communicator_t *comm, mca_coll_base
141141
}
142142

143143
size = ompi_comm_size(comm);
144-
if (size == 1) {
144+
if (1 == size) {
145145
return err;
146146
}
147147
if (!subc->initialized && size > 1) {

0 commit comments

Comments
 (0)