Skip to content

Add calls to MPI-IO large count APIs #191

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Aug 7, 2025
Merged

Add calls to MPI-IO large count APIs #191

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
9999d78
add MPI-IO large-count APIs
wkliao Aug 7, 2025
b5f4318
TRACE_IO name of MPI-IO API called
wkliao Aug 7, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion configure.ac
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1339,7 +1339,11 @@ AC_CHECK_FUNCS([MPI_Type_create_subarray_c \
MPI_Bcast_c \
MPI_Get_count_c \
MPI_Pack_c \
MPI_Unpack_c], [], [have_mpi_large_count_apis=no])
MPI_Unpack_c \
MPI_File_read_at_c \
MPI_File_read_at_all_c \
MPI_File_write_at_c \
MPI_File_write_at_all_c], [], [have_mpi_large_count_apis=no])
# If one of the above APIs is not available, have_mpi_large_count_apis will be
# set to no
UD_MSG_DEBUG([have_mpi_large_count_apis=$have_mpi_large_count_apis])
Expand Down
29 changes: 15 additions & 14 deletions src/drivers/ncmpio/ncmpio_close.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,29 +60,30 @@ ncmpio_free_NC(NC *ncp)
/*----< ncmpio_close_files() >-----------------------------------------------*/
int
ncmpio_close_files(NC *ncp, int doUnlink) {
char *mpi_name;
int mpireturn;

assert(ncp != NULL); /* this should never occur */

if (ncp->independent_fh != MPI_FILE_NULL) {
TRACE_IO(MPI_File_close)(&ncp->independent_fh);
TRACE_IO(MPI_File_close, (&ncp->independent_fh));
if (mpireturn != MPI_SUCCESS)
return ncmpii_error_mpi2nc(mpireturn, "MPI_File_close");
return ncmpii_error_mpi2nc(mpireturn, mpi_name);
}

if (ncp->nprocs > 1 && ncp->collective_fh != MPI_FILE_NULL) {
TRACE_IO(MPI_File_close)(&ncp->collective_fh);
TRACE_IO(MPI_File_close, (&ncp->collective_fh));
if (mpireturn != MPI_SUCCESS)
return ncmpii_error_mpi2nc(mpireturn, "MPI_File_close");
return ncmpii_error_mpi2nc(mpireturn, mpi_name);
}

if (doUnlink) {
/* called from ncmpi_abort, if the file is being created and is still
* in define mode, the file is deleted */
if (ncp->rank == 0) {
TRACE_IO(MPI_File_delete)((char *)ncp->path, ncp->mpiinfo);
TRACE_IO(MPI_File_delete, ((char *)ncp->path, ncp->mpiinfo));
if (mpireturn != MPI_SUCCESS)
return ncmpii_error_mpi2nc(mpireturn, "MPI_File_delete");
return ncmpii_error_mpi2nc(mpireturn, mpi_name);
}
if (ncp->nprocs > 1)
MPI_Barrier(ncp->comm);
Expand Down Expand Up @@ -186,31 +187,31 @@ ncmpio_close(void *ncdp)
#else
MPI_File fh;
int mpireturn;
mpireturn = MPI_File_open(MPI_COMM_SELF, ncp->path, MPI_MODE_RDWR, MPI_INFO_NULL, &fh);
TRACE_IO(MPI_File_open, (MPI_COMM_SELF, ncp->path, MPI_MODE_RDWR, MPI_INFO_NULL, &fh));
if (mpireturn == MPI_SUCCESS) {
/* obtain file size */
MPI_Offset *file_size;
mpireturn = MPI_File_get_size(fh, &file_size);
TRACE_IO(MPI_File_get_size, (fh, &file_size));
if (mpireturn != MPI_SUCCESS) {
err = ncmpii_error_mpi2nc(mpireturn,"MPI_File_get_size");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
if (status == NC_NOERR) status = err;
}
/* truncate file size to header size, if larger than header */
if (file_size > ncp->xsz) {
mpireturn = MPI_File_set_size(fh, ncp->xsz);
TRACE_IO(MPI_File_set_size, (fh, ncp->xsz));
if (mpireturn != MPI_SUCCESS) {
err = ncmpii_error_mpi2nc(mpireturn,"MPI_File_set_size");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
if (status == NC_NOERR) status = err;
}
}
mpireturn = MPI_File_close(&fh);
TRACE_IO(MPI_File_close, (&fh));
if (mpireturn != MPI_SUCCESS) {
err = ncmpii_error_mpi2nc(mpireturn,"MPI_File_close");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
if (status == NC_NOERR) status = err;
}
}
else {
err = ncmpii_error_mpi2nc(mpireturn,"MPI_File_open");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
if (status == NC_NOERR) status = err;
}
#endif
Expand Down
36 changes: 19 additions & 17 deletions src/drivers/ncmpio/ncmpio_create.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ ncmpio_create(MPI_Comm comm,
MPI_Info user_info, /* user's and env info combined */
void **ncpp)
{
char *env_str, *filename;
char *env_str, *filename, *mpi_name;
int rank, nprocs, mpiomode, err, mpireturn, default_format, file_exist=1;
int use_trunc=1;
MPI_File fh;
Expand Down Expand Up @@ -138,25 +138,28 @@ ncmpio_create(MPI_Comm comm,
* expensive
*/
err = unlink(filename);
if (err < 0 && errno != ENOENT) /* ignore ENOENT: file not exist */
if (err < 0 && errno != ENOENT)
/* ignore ENOENT: file not exist */
DEBUG_ASSIGN_ERROR(err, NC_EFILE) /* other error */
else
err = NC_NOERR;
#else
err = NC_NOERR;
TRACE_IO(MPI_File_delete)((char *)path, MPI_INFO_NULL);
TRACE_IO(MPI_File_delete, ((char *)path, MPI_INFO_NULL));
if (mpireturn != MPI_SUCCESS) {
int errorclass;
MPI_Error_class(mpireturn, &errorclass);
if (errorclass != MPI_ERR_NO_SUCH_FILE) /* ignore file not exist */
err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_delete");
if (errorclass != MPI_ERR_NO_SUCH_FILE)
/* ignore file not exist */
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
}
#endif
}
else { /* file is not a regular file, truncate it to zero size */
#ifdef HAVE_TRUNCATE
err = truncate(filename, 0); /* can be expensive */
if (err < 0 && errno != ENOENT) /* ignore ENOENT: file not exist */
if (err < 0 && errno != ENOENT)
/* ignore ENOENT: file not exist */
DEBUG_ASSIGN_ERROR(err, NC_EFILE) /* other error */
else
err = NC_NOERR;
Expand All @@ -174,26 +177,25 @@ ncmpio_create(MPI_Comm comm,
* be expensive.
*/
err = NC_NOERR;
TRACE_IO(MPI_File_open)(MPI_COMM_SELF, (char *)path, MPI_MODE_RDWR,
MPI_INFO_NULL, &fh);
TRACE_IO(MPI_File_open, (MPI_COMM_SELF, (char *)path, MPI_MODE_RDWR, MPI_INFO_NULL, &fh));
if (mpireturn != MPI_SUCCESS) {
int errorclass;
MPI_Error_class(mpireturn, &errorclass);
err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_open");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
}
else {
TRACE_IO(MPI_File_set_size)(fh, 0); /* can be expensive */
TRACE_IO(MPI_File_set_size, (fh, 0)); /* can be expensive */
if (mpireturn != MPI_SUCCESS) {
int errorclass;
MPI_Error_class(mpireturn, &errorclass);
err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_set_size");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
}
else {
TRACE_IO(MPI_File_close)(&fh);
TRACE_IO(MPI_File_close, (&fh));
if (mpireturn != MPI_SUCCESS) {
int errorclass;
MPI_Error_class(mpireturn, &errorclass);
err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_close");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
}
}
}
Expand All @@ -208,7 +210,7 @@ ncmpio_create(MPI_Comm comm,
}

/* create file collectively -------------------------------------------- */
TRACE_IO(MPI_File_open)(comm, (char *)path, mpiomode, user_info, &fh);
TRACE_IO(MPI_File_open, (comm, (char *)path, mpiomode, user_info, &fh));
if (mpireturn != MPI_SUCCESS) {
#ifndef HAVE_ACCESS
if (fIsSet(cmode, NC_NOCLOBBER)) {
Expand All @@ -226,7 +228,7 @@ ncmpio_create(MPI_Comm comm,
if (errno == EEXIST) DEBUG_RETURN_ERROR(NC_EEXIST)
}
#endif
return ncmpii_error_mpi2nc(mpireturn, "MPI_File_open");
return ncmpii_error_mpi2nc(mpireturn, mpi_name);
/* for NC_NOCLOBBER, MPI_MODE_EXCL was added to mpiomode. If the file
* already exists, MPI-IO should return error class MPI_ERR_FILE_EXISTS
* which PnetCDF will return error code NC_EEXIST. This is checked
Expand All @@ -238,9 +240,9 @@ ncmpio_create(MPI_Comm comm,
errno = 0;

/* get the I/O hints used/modified by MPI-IO */
mpireturn = MPI_File_get_info(fh, &info_used);
TRACE_IO(MPI_File_get_info, (fh, &info_used));
if (mpireturn != MPI_SUCCESS)
return ncmpii_error_mpi2nc(mpireturn, "MPI_File_get_info");
return ncmpii_error_mpi2nc(mpireturn, mpi_name);

/* Now the file has been successfully created, allocate/set NC object */

Expand Down
60 changes: 34 additions & 26 deletions src/drivers/ncmpio/ncmpio_enddef.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -167,6 +167,7 @@ move_file_block(NC *ncp,
MPI_Offset from, /* source starting file offset */
MPI_Offset nbytes) /* amount to be moved */
{
char *mpi_name;
int rank, nprocs, mpireturn, err, status=NC_NOERR, do_coll;
void *buf;
size_t num_moves, mv_amnt, p_units;
Expand Down Expand Up @@ -239,14 +240,16 @@ move_file_block(NC *ncp,
mpireturn = MPI_SUCCESS;

/* read from file at off_from for amount of chunk_size */
if (do_coll)
TRACE_IO(MPI_File_read_at_all)(fh, off_from, buf, chunk_size,
MPI_BYTE, &mpistatus);
else if (chunk_size > 0)
TRACE_IO(MPI_File_read_at)(fh, off_from, buf, chunk_size,
MPI_BYTE, &mpistatus);
if (do_coll) {
TRACE_IO(MPI_File_read_at_all, (fh, off_from, buf, chunk_size,
MPI_BYTE, &mpistatus));
}
else if (chunk_size > 0) {
TRACE_IO(MPI_File_read_at, (fh, off_from, buf, chunk_size,
MPI_BYTE, &mpistatus));
}
if (mpireturn != MPI_SUCCESS) {
err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_read_at_all");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
if (status == NC_NOERR && err == NC_EFILE)
DEBUG_ASSIGN_ERROR(status, NC_EREAD)
get_size = chunk_size;
Expand Down Expand Up @@ -282,16 +285,18 @@ move_file_block(NC *ncp,
* call to MPI_Get_count() above returns the actual amount of data read
* from the file, i.e. get_size.
*/
if (do_coll)
TRACE_IO(MPI_File_write_at_all)(fh, off_to, buf,
get_size /* NOT chunk_size */,
MPI_BYTE, &mpistatus);
else if (get_size > 0)
TRACE_IO(MPI_File_write_at)(fh, off_to, buf,
get_size /* NOT chunk_size */,
MPI_BYTE, &mpistatus);
if (do_coll) {
TRACE_IO(MPI_File_write_at_all, (fh, off_to, buf,
get_size /* NOT chunk_size */,
MPI_BYTE, &mpistatus));
}
else if (get_size > 0) {
TRACE_IO(MPI_File_write_at, (fh, off_to, buf,
get_size /* NOT chunk_size */,
MPI_BYTE, &mpistatus));
}
if (mpireturn != MPI_SUCCESS) {
err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_write_at_all");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
if (status == NC_NOERR && err == NC_EFILE)
DEBUG_ASSIGN_ERROR(status, NC_EWRITE)
}
Expand Down Expand Up @@ -597,6 +602,7 @@ NC_begins(NC *ncp)
static int
write_NC(NC *ncp)
{
char *mpi_name;
int status=NC_NOERR, mpireturn, err, is_coll;
MPI_Offset i, header_wlen, ntimes;
MPI_File fh;
Expand Down Expand Up @@ -680,14 +686,16 @@ write_NC(NC *ncp)
buf_ptr = buf;
for (i=0; i<ntimes; i++) {
int bufCount = (int) MIN(remain, NC_MAX_INT);
if (is_coll)
TRACE_IO(MPI_File_write_at_all)(fh, offset, buf_ptr, bufCount,
MPI_BYTE, &mpistatus);
else
TRACE_IO(MPI_File_write_at)(fh, offset, buf_ptr, bufCount,
MPI_BYTE, &mpistatus);
if (is_coll) {
TRACE_IO(MPI_File_write_at_all, (fh, offset, buf_ptr, bufCount,
MPI_BYTE, &mpistatus));
}
else {
TRACE_IO(MPI_File_write_at, (fh, offset, buf_ptr, bufCount,
MPI_BYTE, &mpistatus));
}
if (mpireturn != MPI_SUCCESS) {
err = ncmpii_error_mpi2nc(mpireturn, "MPI_File_write_at");
err = ncmpii_error_mpi2nc(mpireturn, mpi_name);
/* write has failed, which is more serious than inconsistency */
if (err == NC_EFILE) DEBUG_ASSIGN_ERROR(status, NC_EWRITE)
}
Expand All @@ -711,9 +719,9 @@ write_NC(NC *ncp)
}
else if (fIsSet(ncp->flags, NC_HCOLL)) {
/* other processes participate the collective call */
for (i=0; i<ntimes; i++)
TRACE_IO(MPI_File_write_at_all)(fh, 0, NULL, 0, MPI_BYTE,
&mpistatus);
for (i=0; i<ntimes; i++) {
TRACE_IO(MPI_File_write_at_all, (fh, 0, NULL, 0, MPI_BYTE, &mpistatus));
}
}

fn_exit:
Expand Down
Loading
Loading