Skip to content

Commit b41a82c

Browse files
wkliaowkliao
committed
Add command-line option '-i' to use MPI independent I/O
1 parent d6e35f2 commit b41a82c

File tree

5 files changed

+164
-46
lines changed

5 files changed

+164
-46
lines changed

‎benchmarks/FLASH-IO/README.md‎

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,4 @@
1-
#
2-
# Copyright (C) 2013, Northwestern University
3-
# See COPYRIGHT notice in top-level directory.
4-
#
1+
## PnetCDF FLASH-IO Benchmark
52

63
This software benchmarks the performance of PnetCDF method for the I/O and data
74
partitioning pattern from the FLASH I/O benchmark, developed at the Flash
@@ -51,6 +48,10 @@ and the I/O time of which dominates the entire benchmark.
5148
```
5249
(this will use file base name prefix: "/pvfs2/flash_io_test_")
5350

51+
+ Command-line options:
52+
* [-q] quiet mode
53+
* [-i] use MPI independent I/O (default is collective I/O)
54+
5455
* Example output on screen:
5556
```c
5657
number of guards : 4
@@ -97,3 +98,6 @@ and the I/O time of which dominates the entire benchmark.
9798
```
9899
99100
101+
* Copyright (C) 2013, Northwestern University
102+
* See COPYRIGHT notice in top-level directory.
103+

‎benchmarks/FLASH-IO/checkpoint_ncmpi_parallel.F90‎

Lines changed: 62 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -391,6 +391,11 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
391391
MPI_Wtime() - time_start
392392
#endif
393393

394+
if (indep_io) then
395+
err = nfmpi_begin_indep_data(ncid)
396+
if (err .NE. NF_NOERR) call check(err, "nfmpi_begin_indep_data")
397+
endif
398+
394399
global_offset = n_to_left(MyPE)
395400

396401
!-----------------------------------------------------------------------------
@@ -408,8 +413,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
408413
err = nfmpi_iput_vara_int(ncid, varid(1), starts, counts, lrefine, reqs(1))
409414
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: lrefine")
410415
else
411-
err = nfmpi_put_vara_int_all(ncid, varid(1), starts, counts, lrefine)
412-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: lrefine")
416+
if (indep_io) then
417+
err = nfmpi_put_vara_int(ncid, varid(1), starts, counts, lrefine)
418+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: lrefine")
419+
else
420+
err = nfmpi_put_vara_int_all(ncid, varid(1), starts, counts, lrefine)
421+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: lrefine")
422+
endif
413423
endif
414424

415425
#ifdef TIMERS
@@ -426,8 +436,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
426436
err = nfmpi_iput_vara_int(ncid, varid(2), starts, counts, nodetype, reqs(2))
427437
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: nodetype")
428438
else
429-
err = nfmpi_put_vara_int_all(ncid, varid(2), starts, counts, nodetype)
430-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: nodetype")
439+
if (indep_io) then
440+
err = nfmpi_put_vara_int(ncid, varid(2), starts, counts, nodetype)
441+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: nodetype")
442+
else
443+
err = nfmpi_put_vara_int_all(ncid, varid(2), starts, counts, nodetype)
444+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: nodetype")
445+
endif
431446
endif
432447

433448
#ifdef TIMERS
@@ -448,8 +463,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
448463
err = nfmpi_iput_vara_int(ncid, varid(3), starts, counts, gid, reqs(3))
449464
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: gid")
450465
else
451-
err = nfmpi_put_vara_int_all(ncid, varid(3), starts, counts, gid)
452-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: gid")
466+
if (indep_io) then
467+
err = nfmpi_put_vara_int(ncid, varid(3), starts, counts, gid)
468+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: gid")
469+
else
470+
err = nfmpi_put_vara_int_all(ncid, varid(3), starts, counts, gid)
471+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: gid")
472+
endif
453473
endif
454474

455475
#ifdef TIMERS
@@ -475,8 +495,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
475495
err = nfmpi_iput_vara_double(ncid, varid(4), starts, counts, coord_buf, reqs(4))
476496
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_double: coord")
477497
else
478-
err = nfmpi_put_vara_double_all(ncid, varid(4), starts, counts, coord_buf)
479-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: coord")
498+
if (indep_io) then
499+
err = nfmpi_put_vara_double(ncid, varid(4), starts, counts, coord_buf)
500+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double: coord")
501+
else
502+
err = nfmpi_put_vara_double_all(ncid, varid(4), starts, counts, coord_buf)
503+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: coord")
504+
endif
480505
endif
481506

482507
#ifdef TIMERS
@@ -498,8 +523,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
498523
err = nfmpi_iput_vara_double(ncid, varid(5), starts, counts, bs_buf, reqs(5))
499524
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_double: size")
500525
else
501-
err = nfmpi_put_vara_double_all(ncid, varid(5), starts, counts, bs_buf)
502-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: size")
526+
if (indep_io) then
527+
err = nfmpi_put_vara_double(ncid, varid(5), starts, counts, bs_buf)
528+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double: size")
529+
else
530+
err = nfmpi_put_vara_double_all(ncid, varid(5), starts, counts, bs_buf)
531+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: size")
532+
endif
503533
endif
504534

505535
#ifdef TIMERS
@@ -523,8 +553,13 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
523553
err = nfmpi_iput_vara_double(ncid, varid(6), starts, counts, bb_buf, reqs(6))
524554
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_double: bnd_box")
525555
else
526-
err = nfmpi_put_vara_double_all(ncid, varid(6), starts, counts, bb_buf)
527-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: bnd_box")
556+
if (indep_io) then
557+
err = nfmpi_put_vara_double(ncid, varid(6), starts, counts, bb_buf)
558+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double: bnd_box")
559+
else
560+
err = nfmpi_put_vara_double_all(ncid, varid(6), starts, counts, bb_buf)
561+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: bnd_box")
562+
endif
528563
endif
529564

530565
#ifdef TIMERS
@@ -595,23 +630,31 @@ double precision function checkpoint_wr_ncmpi_par (filenum, simtime)
595630
if (err .NE. NF_NOERR) &
596631
call check(err, "nfmpi_iput_vara: unknowns")
597632
else
598-
err = nfmpi_put_vara_double_all(ncid, varid(6+i), starts, counts, unk_buf)
599-
if (err .NE. NF_NOERR) &
600-
call check(err, "nfmpi_put_vara_double_all: unknowns")
633+
if (indep_io) then
634+
err = nfmpi_put_vara_double(ncid, varid(6+i), starts, counts, unk_buf)
635+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double: unknowns")
636+
else
637+
err = nfmpi_put_vara_double_all(ncid, varid(6+i), starts, counts, unk_buf)
638+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_double_all: unknowns")
639+
endif
601640
endif
602641
enddo
603642

604643
if (use_nonblocking_io) then
605644
! wait for the nonblocking I/O to complete
606-
err = nfmpi_wait_all(ncid, nvar+6, reqs, stats)
607-
if (err .NE. NF_NOERR) &
608-
call check(err, "nfmpi_wait_all: unknowns")
645+
if (indep_io) then
646+
err = nfmpi_wait(ncid, nvar+6, reqs, stats)
647+
if (err .NE. NF_NOERR) call check(err, "nfmpi_wait: unknowns")
648+
else
649+
err = nfmpi_wait_all(ncid, nvar+6, reqs, stats)
650+
if (err .NE. NF_NOERR) call check(err, "nfmpi_wait_all: unknowns")
651+
endif
609652

610653
! check the status of each nonblocking request
611654
do i=1, nvar+6
612655
write(str,'(I2)') i
613656
if (stats(i) .NE. NF_NOERR) &
614-
call check(stats(i), 'In nfmpi_wait_all req '//trim(str))
657+
call check(stats(i), 'In nfmpi_wait(_all) req '//trim(str))
615658
enddo
616659
call MPI_Type_free(buftype, err)
617660
endif

‎benchmarks/FLASH-IO/common.fh‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -49,8 +49,10 @@
4949

5050
integer info_used
5151
logical use_nonblocking_io
52+
logical indep_io
5253

5354
double precision chk_t(3), corner_t(3), nocorner_t(3)
54-
common /timers/ chk_t, corner_t, nocorner_t, info_used, use_nonblocking_io
55+
common /timers/ chk_t, corner_t, nocorner_t, info_used, &
56+
use_nonblocking_io, indep_io
5557

5658

‎benchmarks/FLASH-IO/flash_benchmark_io.F90‎

Lines changed: 22 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@ program flash_benchmark_io
4141

4242
MasterPE = 0
4343
verbose = .TRUE.
44+
indep_io = .FALSE.
4445

4546
! root process reads command-line arguments
4647
if (MyPE .EQ. MasterPE) then
@@ -55,13 +56,28 @@ program flash_benchmark_io
5556
if (basenm(1:2) .EQ. '-q') then
5657
verbose = .FALSE.
5758
basenm = "flash_io_test_"
59+
else if (basenm(1:2) .EQ. '-i') then
60+
indep_io = .TRUE.
61+
basenm = "flash_io_test_"
5862
endif
5963
else if (argc .EQ. 2) then
60-
verbose = .FALSE.
61-
call getarg(2, basenm)
64+
call getarg(1, basenm)
6265
if (basenm(1:2) .EQ. '-q') then
66+
verbose = .FALSE.
67+
call getarg(2, basenm)
68+
else if (basenm(1:2) .EQ. '-i') then
69+
indep_io = .TRUE.
70+
call getarg(2, basenm)
71+
else
72+
isArgvRight = .FALSE.
73+
endif
74+
else if (argc .EQ. 3) then
75+
call getarg(3, basenm)
76+
if (basenm(1:2) .EQ. '-q' .OR. basenm(1:2) .EQ. '-i') then
6377
isArgvRight = .FALSE.
6478
endif
79+
verbose = .FALSE.
80+
indep_io = .TRUE.
6581
else if (argc .GT. 2) then
6682
isArgvRight = .FALSE.
6783
endif
@@ -76,6 +92,10 @@ program flash_benchmark_io
7692
MPI_COMM_WORLD, ierr)
7793
if (.NOT. isArgvRight) goto 999
7894

95+
! broadcast if independent I/O should be used
96+
call MPI_Bcast(indep_io, 1, MPI_LOGICAL, MasterPE, &
97+
MPI_COMM_WORLD, ierr)
98+
7999
! broadcast file base name prefix
80100
call MPI_Bcast(basenm, 128, MPI_CHARACTER, MasterPE, &
81101
MPI_COMM_WORLD, ierr)

‎benchmarks/FLASH-IO/plotfile_ncmpi_parallel.F90‎

Lines changed: 69 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,11 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
430430
sunklabels, &
431431
varid)
432432

433+
if (indep_io) then
434+
err = nfmpi_begin_indep_data(ncid)
435+
if (err .NE. NF_NOERR) call check(err, "nfmpi_begin_indep_data")
436+
endif
437+
433438
global_offset = n_to_left(MyPE)
434439
!-----------------------------------------------------------------------------
435440
! store the tree information
@@ -442,17 +447,27 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
442447
err = nfmpi_iput_vara_int(ncid, varid(1), starts, counts, lrefine, reqs(1))
443448
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: lrefine sp")
444449
else
445-
err = nfmpi_put_vara_int_all(ncid, varid(1), starts, counts, lrefine)
446-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: lrefine sp")
450+
if (indep_io) then
451+
err = nfmpi_put_vara_int(ncid, varid(1), starts, counts, lrefine)
452+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: lrefine sp")
453+
else
454+
err = nfmpi_put_vara_int_all(ncid, varid(1), starts, counts, lrefine)
455+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: lrefine sp")
456+
endif
447457
endif
448458

449459
! store the nodetype
450460
if (use_nonblocking_io) then
451461
err = nfmpi_iput_vara_int(ncid, varid(2), starts, counts, nodetype, reqs(2))
452462
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: nodetype sp")
453463
else
454-
err = nfmpi_put_vara_int_all(ncid, varid(2), starts, counts, nodetype)
455-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: nodetype sp")
464+
if (indep_io) then
465+
err = nfmpi_put_vara_int(ncid, varid(2), starts, counts, nodetype)
466+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: nodetype sp")
467+
else
468+
err = nfmpi_put_vara_int_all(ncid, varid(2), starts, counts, nodetype)
469+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: nodetype sp")
470+
endif
456471
endif
457472

458473
! store the global id
@@ -464,8 +479,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
464479
err = nfmpi_iput_vara_int(ncid, varid(3), starts, counts, gid, reqs(3))
465480
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_int: gid sp")
466481
else
467-
err = nfmpi_put_vara_int_all(ncid, varid(3), starts, counts, gid)
468-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: gid sp")
482+
if (indep_io) then
483+
err = nfmpi_put_vara_int(ncid, varid(3), starts, counts, gid)
484+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int: gid sp")
485+
else
486+
err = nfmpi_put_vara_int_all(ncid, varid(3), starts, counts, gid)
487+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_int_all: gid sp")
488+
endif
469489
endif
470490

471491
!-----------------------------------------------------------------------------
@@ -484,8 +504,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
484504
err = nfmpi_iput_vara_real(ncid, varid(4), starts, counts, coord_single, reqs(4))
485505
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_real: coord sp")
486506
else
487-
err = nfmpi_put_vara_real_all(ncid, varid(4), starts, counts, coord_single)
488-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_read_all: coord sp")
507+
if (indep_io) then
508+
err = nfmpi_put_vara_real(ncid, varid(4), starts, counts, coord_single)
509+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_read: coord sp")
510+
else
511+
err = nfmpi_put_vara_real_all(ncid, varid(4), starts, counts, coord_single)
512+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_read_all: coord sp")
513+
endif
489514
endif
490515

491516
! store the block size
@@ -501,8 +526,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
501526
err = nfmpi_iput_vara_real(ncid, varid(5), starts, counts, blk_sz_single, reqs(5))
502527
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_real: size sp")
503528
else
504-
err = nfmpi_put_vara_real_all(ncid, varid(5), starts, counts, blk_sz_single)
505-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: size sp")
529+
if (indep_io) then
530+
err = nfmpi_put_vara_real(ncid, varid(5), starts, counts, blk_sz_single)
531+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real: size sp")
532+
else
533+
err = nfmpi_put_vara_real_all(ncid, varid(5), starts, counts, blk_sz_single)
534+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: size sp")
535+
endif
506536
endif
507537

508538
! store the bounding box
@@ -522,8 +552,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
522552
err = nfmpi_iput_vara_real(ncid, varid(6), starts, counts, bnd_single, reqs(6))
523553
if (err .NE. NF_NOERR) call check(err, "nfmpi_iput_vara_real: bnd_box")
524554
else
525-
err = nfmpi_put_vara_real_all(ncid, varid(6), starts, counts, bnd_single)
526-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: bnd_box")
555+
if (indep_io) then
556+
err = nfmpi_put_vara_real(ncid, varid(6), starts, counts, bnd_single)
557+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real: bnd_box")
558+
else
559+
err = nfmpi_put_vara_real_all(ncid, varid(6), starts, counts, bnd_single)
560+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: bnd_box")
561+
endif
527562
endif
528563

529564
if (use_nonblocking_io) then
@@ -614,8 +649,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
614649
err = nfmpi_bput_vara_real(ncid, varid(6+ivar), starts, counts, unkt_crn, reqs(ivar+6))
615650
if (err .NE. NF_NOERR) call check(err, "nfmpi_bput_vara_real: unknowns sp")
616651
else
617-
err = nfmpi_put_vara_real_all(ncid, varid(6+ivar), starts, counts, unkt_crn)
618-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: unknowns sp")
652+
if (indep_io) then
653+
err = nfmpi_put_vara_real(ncid, varid(6+ivar), starts, counts, unkt_crn)
654+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real: unknowns sp")
655+
else
656+
err = nfmpi_put_vara_real_all(ncid, varid(6+ivar), starts, counts, unkt_crn)
657+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: unknowns sp")
658+
endif
619659
endif
620660

621661
else
@@ -638,8 +678,13 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
638678
err = nfmpi_bput_vara_real(ncid, varid(6+ivar), starts, counts, unkt, reqs(ivar+6))
639679
if (err .NE. NF_NOERR) call check(err, "nfmpi_bput_vara_real: unknowns sp")
640680
else
641-
err = nfmpi_put_vara_real_all(ncid, varid(6+ivar), starts, counts, unkt)
642-
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: unknowns sp")
681+
if (indep_io) then
682+
err = nfmpi_put_vara_real(ncid, varid(6+ivar), starts, counts, unkt)
683+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real: unknowns sp")
684+
else
685+
err = nfmpi_put_vara_real_all(ncid, varid(6+ivar), starts, counts, unkt)
686+
if (err .NE. NF_NOERR) call check(err, "nfmpi_put_vara_real_all: unknowns sp")
687+
endif
643688
endif
644689
endif
645690

@@ -648,15 +693,19 @@ double precision function plotfile_ncmpi_par(filenum, simtime, corners)
648693
! wait for all nonblocking requests to complete
649694
if (use_nonblocking_io) then
650695
! wait for the nonblocking I/O to complete
651-
err = nfmpi_wait_all(ncid, num_out+6, reqs, stats)
652-
if (err .NE. NF_NOERR) &
653-
call check(err, "(sp) nfmpi_wait_all: ")
696+
if (indep_io) then
697+
err = nfmpi_wait(ncid, num_out+6, reqs, stats)
698+
if (err .NE. NF_NOERR) call check(err, "(sp) nfmpi_wait: ")
699+
else
700+
err = nfmpi_wait_all(ncid, num_out+6, reqs, stats)
701+
if (err .NE. NF_NOERR) call check(err, "(sp) nfmpi_wait_all: ")
702+
endif
654703

655704
! check the status of each nonblocking request
656705
do i=1, num_out+6
657706
write(str,'(I2)') i
658707
if (stats(i) .NE. NF_NOERR) &
659-
call check(stats(i), '(sp) nfmpi_wait_all req '//trim(str))
708+
call check(stats(i), '(sp) nfmpi_wait(_all) req '//trim(str))
660709
enddo
661710

662711
! detach the temporary buffer

0 commit comments

Comments
 (0)