Skip to content

Commit 1b0bf0f

Browse files
wkliaowkliao
committed
add a new MPI program to evaluate MPI_Alltoallw
1 parent 5e95f0e commit 1b0bf0f

File tree

2 files changed

+427
-0
lines changed

2 files changed

+427
-0
lines changed

‎MPI/alltoallw.c‎

Lines changed: 356 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,356 @@
1+
/* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * *
2+
*
3+
* Copyright (C) 2025, Northwestern University
4+
* See COPYRIGHT notice in top-level directory.
5+
*
6+
* Evaluate performane of all-to-many personalized communication implemented
7+
* with MPI_Alltoallw() and MPI_Issend()/MPI_Irecv().
8+
*
9+
* To compile:
10+
* % mpicc -O2 alltoallw.c -o alltoallw
11+
*
12+
* Usage:
13+
* % ./alltoallw -h
14+
* Usage: ./alltoallw [OPTION]
15+
* [-h] Print this help message
16+
* [-v] Verbose mode (default: no)
17+
* [-n num] number of iterations (default: 1)
18+
* [-r ratio] every ratio processes is a receiver (default: 1)
19+
* [-l len] receive message size per iteration( default: 8MB)
20+
*
21+
* Example run command and output on screen:
22+
* % mpiexec -n 2048 ./alltoallw -n 253 -r 32
23+
*
24+
* nprocs = 2048
25+
* ntimes = 253
26+
* num_recvers = 64
27+
* individual message len = 4096 bytes
28+
* send/recv buffer gap = 4 bytes
29+
* Time for using MPI_alltoallw = 53.60 sec
30+
* Time for using MPI_Issend/Irecv = 2.59 sec
31+
*
32+
* * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * * */
33+
34+
#include <stdio.h>
35+
#include <stdlib.h>
36+
#include <string.h>
37+
#include <unistd.h>
38+
#include <assert.h>
39+
40+
#include <mpi.h>
41+
42+
static int verbose;
43+
44+
#define GAP 4
45+
46+
#define ERR \
47+
if (err != MPI_SUCCESS) { \
48+
int errorStringLen; \
49+
char errorString[MPI_MAX_ERROR_STRING]; \
50+
MPI_Error_string(err, errorString, &errorStringLen); \
51+
printf("Error at line %d: %s\n",__LINE__,errorString); \
52+
goto err_out; \
53+
}
54+
55+
void run_alltoallw(int ntimes,
56+
int ratio,
57+
int is_receiver,
58+
int len,
59+
int *sendBuf,
60+
int *recvBuf)
61+
{
62+
int *sendPtr, *recvPtr;
63+
int i, j, err, nprocs, rank, num_recvers;
64+
int *sendCounts, *recvCounts, *sendDisps, *recvDisps;
65+
MPI_Datatype *sendTypes, *recvTypes;
66+
double timing, maxt;
67+
68+
MPI_Barrier(MPI_COMM_WORLD);
69+
timing = MPI_Wtime();
70+
71+
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
72+
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
73+
num_recvers = nprocs/ ratio;
74+
75+
sendTypes = (MPI_Datatype*) malloc(sizeof(MPI_Datatype) * nprocs * 2);
76+
recvTypes = sendTypes + nprocs;
77+
for (i=0; i<nprocs * 2; i++) sendTypes[i] = MPI_INT;
78+
79+
sendCounts = (int*) calloc(nprocs * 2, sizeof(int));
80+
recvCounts = sendCounts + nprocs;
81+
sendDisps = (int*) calloc(nprocs * 2, sizeof(int));
82+
recvDisps = sendDisps + nprocs;
83+
84+
sendPtr = sendBuf;
85+
recvPtr = recvBuf;
86+
87+
/* Only receivers has non-zero data to receive */
88+
if (is_receiver) {
89+
j = 0;
90+
for (i=0; i<nprocs; i++) {
91+
if (i != rank) { /* skip receiving from self */
92+
recvCounts[i] = len;
93+
recvDisps[i] = (len + GAP) * j * sizeof(int);
94+
}
95+
j++;
96+
if (verbose && i != rank)
97+
printf("%2d recv from %2d of %d\n",rank,i,recvCounts[i]);
98+
}
99+
}
100+
101+
/* All ranks send to each receivers */
102+
j = 0;
103+
for (i=0; i<nprocs; i++) {
104+
if (i % ratio) continue; /* i is not a receiver */
105+
if (i != rank) { /* skip sending to self */
106+
sendCounts[i] = len;
107+
sendDisps[i] = (len + GAP) * j * sizeof(int);
108+
}
109+
j++;
110+
if (verbose && i != rank)
111+
printf("%2d send to %2d of %d\n",rank,i,sendCounts[i]);
112+
}
113+
114+
for (i=0; i<ntimes; i++) {
115+
err = MPI_Alltoallw(sendPtr, sendCounts, sendDisps, sendTypes,
116+
recvPtr, recvCounts, recvDisps, recvTypes,
117+
MPI_COMM_WORLD); ERR
118+
sendPtr += num_recvers * (len + GAP);
119+
recvPtr += nprocs * (len + GAP);
120+
}
121+
122+
err_out:
123+
free(sendTypes);
124+
free(sendCounts);
125+
free(sendDisps);
126+
127+
timing = MPI_Wtime() - timing;
128+
MPI_Reduce(&timing, &maxt, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
129+
if (rank == 0)
130+
printf("Time for using MPI_alltoallw = %.2f sec\n", maxt);
131+
}
132+
133+
void run_async_send_recv(int ntimes,
134+
int ratio,
135+
int is_receiver,
136+
int len,
137+
int *sendBuf,
138+
int *recvBuf)
139+
{
140+
int *sendPtr, *recvPtr;
141+
int i, j, err, nprocs, rank, nreqs, num_recvers;
142+
MPI_Request *reqs;
143+
MPI_Status *st;
144+
double timing, maxt;
145+
146+
MPI_Barrier(MPI_COMM_WORLD);
147+
timing = MPI_Wtime();
148+
149+
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
150+
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
151+
num_recvers = nprocs/ ratio;
152+
153+
/* allocate MPI_Request and MPI_Status arrays */
154+
reqs = (MPI_Request*) malloc(sizeof(MPI_Request) * (nprocs + num_recvers));
155+
st = (MPI_Status*) malloc(sizeof(MPI_Status) * (nprocs + num_recvers));
156+
157+
sendPtr = sendBuf;
158+
recvPtr = recvBuf;
159+
160+
for (i=0; i<ntimes; i++) {
161+
nreqs = 0;
162+
163+
/* Only receivers post recv requests */
164+
if (is_receiver) {
165+
for (j=0; j<nprocs; j++) {
166+
if (rank != j) { /* skip recv from self */
167+
err = MPI_Irecv(recvPtr, len, MPI_INT, j, 0, MPI_COMM_WORLD,
168+
&reqs[nreqs++]);
169+
ERR
170+
}
171+
recvPtr += len + GAP;
172+
}
173+
}
174+
175+
/* all ranks post send requests */
176+
for (j=0; j<nprocs; j++) {
177+
if (j % ratio) continue; /* j is not a receiver */
178+
if (rank != j) { /* skip send to self */
179+
err = MPI_Issend(sendPtr, len, MPI_INT, j, 0, MPI_COMM_WORLD,
180+
&reqs[nreqs++]);
181+
ERR
182+
}
183+
sendPtr += len + GAP;
184+
}
185+
186+
err = MPI_Waitall(nreqs, reqs, st); ERR
187+
}
188+
189+
err_out:
190+
free(st);
191+
free(reqs);
192+
193+
timing = MPI_Wtime() - timing;
194+
MPI_Reduce(&timing, &maxt, 1, MPI_DOUBLE, MPI_MAX, 0, MPI_COMM_WORLD);
195+
if (rank == 0)
196+
printf("Time for using MPI_Issend/Irecv = %.2f sec\n", maxt);
197+
}
198+
199+
void initialize_bufs(int ntimes,
200+
int num_recvers,
201+
int len,
202+
int *sendBuf,
203+
int *recvBuf)
204+
{
205+
int i, j, k, m, nprocs, rank;
206+
207+
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
208+
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
209+
210+
for (i=0; i<(len + GAP)*ntimes*num_recvers; i++)
211+
sendBuf[i] = -2;
212+
for (i=0; i<(len + GAP)*ntimes*nprocs; i++)
213+
recvBuf[i] = -3;
214+
m = 0;
215+
for (i=0; i<ntimes; i++) {
216+
for (j=0; j<num_recvers; j++) {
217+
for (k=0; k<len; k++) {
218+
sendBuf[m++] = rank;
219+
}
220+
m += GAP;
221+
}
222+
}
223+
}
224+
225+
int check_recv_buf(char *comm_op,
226+
int ntimes,
227+
int len,
228+
int *recvBuf)
229+
{
230+
int i, j, k, m, expect, err=0, nprocs, rank;
231+
232+
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
233+
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
234+
235+
m = 0;
236+
for (i=0; i<ntimes; i++) {
237+
for (j=0; j<nprocs; j++) {
238+
for (k=0; k<len+GAP; k++) {
239+
expect = (j == rank) ? -3 : ((k < len) ? j : -3);
240+
if (recvBuf[m] != expect) {
241+
printf("Error(%s): rank %d i=%d j=%d k=%d expect %d but got %d\n",
242+
comm_op, rank, i, j, k, expect, recvBuf[m]);
243+
goto err_out;
244+
}
245+
m++;
246+
}
247+
}
248+
}
249+
err_out:
250+
return err;
251+
}
252+
253+
/*----< usage() >------------------------------------------------------------*/
254+
static void usage (char *argv0) {
255+
char *help = "Usage: %s [OPTION]\n\
256+
[-h] Print this help message\n\
257+
[-v] Verbose mode (default: no)\n\
258+
[-n num] number of iterations (default: 1)\n\
259+
[-r ratio] every ratio processes is a receiver (default: 1)\n\
260+
[-l len] receive message size per iteration( default: 8MB)\n";
261+
fprintf (stderr, help, argv0);
262+
}
263+
264+
/*----< main() >------------------------------------------------------------*/
265+
int main(int argc, char **argv) {
266+
extern int optind;
267+
extern char *optarg;
268+
int i, rank, nprocs;
269+
int len, block_len, ntimes, ratio, num_recvers, is_receiver;
270+
int *sendBuf, *recvBuf;
271+
272+
MPI_Init(&argc, &argv);
273+
MPI_Comm_size(MPI_COMM_WORLD, &nprocs);
274+
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
275+
276+
verbose = 0;
277+
block_len = 2 * 1024 * 1024;
278+
ntimes = 1;
279+
ratio = 1;
280+
281+
/* command-line arguments */
282+
while ((i = getopt (argc, argv, "hvl:n:r:")) != EOF)
283+
switch (i) {
284+
case 'v':
285+
verbose = 1;
286+
break;
287+
case 'l':
288+
block_len = atoi(optarg) / sizeof(int);
289+
break;
290+
case 'n':
291+
ntimes = atoi(optarg);
292+
break;
293+
case 'r':
294+
ratio = atoi(optarg);
295+
break;
296+
case 'h':
297+
default:
298+
if (rank == 0) usage(argv[0]);
299+
goto err_out;
300+
}
301+
302+
/* set the number of receivers */
303+
if (ratio <= 0 || ratio > nprocs) ratio = 1;
304+
num_recvers = nprocs / ratio;
305+
306+
/* set whether this rank has non-zero data to receive */
307+
is_receiver = (rank % ratio == 0) ? 1 : 0;
308+
309+
/* per message size */
310+
len = block_len / nprocs;
311+
312+
if (verbose && rank == 0)
313+
printf("nprocs=%d ntimes=%d block_len=%d num_recvers=%d len=%d\n",
314+
nprocs, ntimes, block_len, num_recvers, len);
315+
316+
if (verbose && is_receiver)
317+
printf("rank %2d is_receiver\n", rank);
318+
319+
if (verbose) fflush(stdout);
320+
321+
if (rank == 0) {
322+
printf("nprocs = %d\n", nprocs);
323+
printf("ntimes = %d\n", ntimes);
324+
printf("num_recvers = %d\n", num_recvers);
325+
printf("individual message len = %zd bytes\n",len*sizeof(int));
326+
printf("send/recv buffer gap = %zd bytes\n",GAP*sizeof(int));
327+
}
328+
329+
/* allocate and initialize send and recevive buffer */
330+
sendBuf = (int*) malloc(sizeof(int) * (len + GAP) * ntimes * num_recvers);
331+
recvBuf = (int*) malloc(sizeof(int) * (len + GAP) * ntimes * nprocs);
332+
333+
initialize_bufs(ntimes, num_recvers, len, sendBuf, recvBuf);
334+
335+
MPI_Barrier(MPI_COMM_WORLD);
336+
run_alltoallw(ntimes, ratio, is_receiver, len, sendBuf, recvBuf);
337+
338+
if (is_receiver)
339+
check_recv_buf("alltoallw", ntimes, len, recvBuf);
340+
341+
initialize_bufs(ntimes, num_recvers, len, sendBuf, recvBuf);
342+
343+
MPI_Barrier(MPI_COMM_WORLD);
344+
run_async_send_recv(ntimes, ratio, is_receiver, len, sendBuf, recvBuf);
345+
346+
if (is_receiver)
347+
check_recv_buf("isend/irecv", ntimes, len, recvBuf);
348+
349+
free(recvBuf);
350+
free(sendBuf);
351+
352+
err_out:
353+
MPI_Finalize();
354+
return 0;
355+
}
356+

0 commit comments

Comments
 (0)