add MPI_File_fence

adammoody · adammoody · commit 41d607cd3969 · 2022-02-10T15:49:58.000-08:00
diff --git a/src/mpi/romio/adio/ad_lustre/ad_lustre.c b/src/mpi/romio/adio/ad_lustre/ad_lustre.c
@@ -32,6 +32,7 @@ struct ADIOI_Fns_struct ADIO_LUSTRE_operations = {
     ADIOI_GEN_IreadStrided,     /* IreadStrided */
     ADIOI_GEN_IwriteStrided,    /* IwriteStrided */
     ADIOI_GEN_Flush,    /* Flush */
+    ADIOI_GEN_Fence,    /* Fence */
     ADIOI_GEN_Resize,   /* Resize */
     ADIOI_GEN_Delete,   /* Delete */
     ADIOI_GEN_Feature,  /* Features */
diff --git a/src/mpi/romio/adio/ad_unify/Makefile.mk b/src/mpi/romio/adio/ad_unify/Makefile.mk
@@ -15,6 +15,7 @@ romio_other_sources +=                          \
     adio/ad_unify/ad_unify_close.c \
     adio/ad_unify/ad_unify_io.c \
     adio/ad_unify/ad_unify_flush.c \
+    adio/ad_unify/ad_unify_fence.c \
     adio/ad_unify/ad_unify_delete.c \
     adio/ad_unify/ad_unify_fcntl.c \
     adio/ad_unify/ad_unify_resize.c \
diff --git a/src/mpi/romio/adio/ad_unify/ad_unify.c b/src/mpi/romio/adio/ad_unify/ad_unify.c
@@ -35,6 +35,7 @@ struct ADIOI_Fns_struct ADIO_UNIFY_operations = {
     ADIOI_FAKE_IreadStrided,    /* IreadStrided */
     ADIOI_FAKE_IwriteStrided,   /* IwriteStrided */
     ADIOI_UNIFY_Flush,  /* Flush */
+    ADIOI_UNIFY_Fence,  /* Fence */
     ADIOI_UNIFY_Resize, /* Resize */
     ADIOI_UNIFY_Delete, /* Delete */
     ADIOI_UNIFY_Feature,
diff --git a/src/mpi/romio/adio/ad_unify/ad_unify_fence.c b/src/mpi/romio/adio/ad_unify/ad_unify_fence.c
@@ -0,0 +1,35 @@
+
+#include "ad_unify.h"
+#include "ad_unify_common.h"
+
+void ADIOI_UNIFY_Fence(ADIO_File fd, int *error_code)
+{
+    int ret;
+    *error_code = MPI_SUCCESS;
+    ADIOI_UNIFY_fs *unifyfs_blob = fd->fs_ptr;
+
+    /* Synchronize with file system to flush writes
+     * to make data visible to other processes. */
+    unifyfs_io_request sync_req = {
+        .op = UNIFYFS_IOREQ_OP_SYNC_META,
+        .gfid = unifyfs_blob->gfid,
+        .result.error = 0,
+    };
+    ret = unifyfs_dispatch_io(unifyfs_blob->fshdl, 1, &sync_req);
+    if (ret == UNIFYFS_SUCCESS) {
+        ret = unifyfs_wait_io(unifyfs_blob->fshdl, 1, &sync_req, 1);
+        if (ret == UNIFYFS_SUCCESS)
+            if (sync_req.result.error != 0) {
+                *error_code = MPIO_Err_create_code(MPI_SUCCESS,
+                                                   MPIR_ERR_RECOVERABLE, __func__, __LINE__,
+                                                   MPI_ERR_IO, "**io", "**io %s",
+                                                   strerror(sync_req.result.error));
+            }
+    }
+
+    /* Synchronize processes so that upon returning from fence,
+     * all procs know that all other procs have completed their flush. */
+    MPI_Barrier(fd->comm);
+
+    return;
+}
diff --git a/src/mpi/romio/adio/common/Makefile.mk b/src/mpi/romio/adio/common/Makefile.mk
@@ -17,6 +17,7 @@ romio_other_sources +=                  \
     adio/common/ad_fcntl.c              \
     adio/common/ad_features.c           \
     adio/common/ad_flush.c              \
+    adio/common/ad_fence.c              \
     adio/common/ad_fstype.c             \
     adio/common/ad_get_sh_fp.c          \
     adio/common/ad_hints.c              \
diff --git a/src/mpi/romio/adio/common/ad_fence.c b/src/mpi/romio/adio/common/ad_fence.c
@@ -0,0 +1,30 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "adio.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+void ADIOI_GEN_Fence(ADIO_File fd, int *error_code)
+{
+    int err;
+    static char myname[] = "ADIOI_GEN_FENCE";
+
+    /* For the general (common) case, we assume that the underlying
+     * file system implements a POSIX-style write,
+     * meaning that written data becomes visible immediately after
+     * returning from a successful write call.
+     *
+     * For such file systems, there is no need to explicitly flush
+     * data or to synchronize with the file system to expose those writes. */
+
+    /* Even though the data flush step is a NOP,
+     * fence semantics still require that we synchronize processes. */
+    MPI_Barrier(fd->comm);
+
+    *error_code = MPI_SUCCESS;
+}
diff --git a/src/mpi/romio/adio/include/adio.h b/src/mpi/romio/adio/include/adio.h
@@ -396,6 +396,7 @@ void ADIO_IwriteStridedColl(ADIO_File fd, void *buf, int count,
 ADIO_Offset ADIO_SeekIndividual(ADIO_File fd, ADIO_Offset offset, int whence, int *error_code);
 void ADIO_Delete(char *filename, int *error_code);
 void ADIO_Flush(ADIO_File fd, int *error_code);
+void ADIO_Fence(ADIO_File fd, int *error_code);
 void ADIO_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
 void ADIO_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
 void ADIO_ResolveFileType(MPI_Comm comm, const char *filename, int *fstype,
diff --git a/src/mpi/romio/adio/include/adioi.h b/src/mpi/romio/adio/include/adioi.h
@@ -332,6 +332,8 @@ struct ADIOI_Fns_struct {
 
 #define ADIO_Flush(fd,error_code) (*(fd->fns->ADIOI_xxx_Flush))(fd,error_code)
 
+#define ADIO_Fence(fd,error_code) (*(fd->fns->ADIOI_xxx_Fence))(fd,error_code)
+
 #define ADIO_Resize(fd,size,error_code)                 \
     (*(fd->fns->ADIOI_xxx_Resize))(fd,size,error_code)
 
diff --git a/src/mpi/romio/include/mpio.h.in b/src/mpi/romio/include/mpio.h.in
@@ -246,6 +246,7 @@ int MPI_Register_datarep(const char *datarep, MPI_Datarep_conversion_function *r
 int MPI_File_set_atomicity(MPI_File fh, int flag) ROMIO_API_PUBLIC;
 int MPI_File_get_atomicity(MPI_File fh, int *flag) ROMIO_API_PUBLIC;
 int MPI_File_sync(MPI_File fh) ROMIO_API_PUBLIC;
+int MPI_File_fence(MPI_File fh) ROMIO_API_PUBLIC;
 
 /* Section 4.13.3 */
 #ifndef MPICH
@@ -549,6 +550,7 @@ int PMPI_Register_datarep(const char *,
 int PMPI_File_set_atomicity(MPI_File, int) ROMIO_API_PUBLIC;
 int PMPI_File_get_atomicity(MPI_File, int *) ROMIO_API_PUBLIC;
 int PMPI_File_sync(MPI_File) ROMIO_API_PUBLIC;
+int PMPI_File_fence(MPI_File) ROMIO_API_PUBLIC;
 
 /* Section 4.13.3 */
 #ifndef MPICH
diff --git a/src/mpi/romio/mpi-io/Makefile.mk b/src/mpi/romio/mpi-io/Makefile.mk
@@ -15,6 +15,7 @@ romio_mpi_sources +=          \
     mpi-io/file_c2f.c         \
     mpi-io/file_f2c.c         \
     mpi-io/fsync.c            \
+    mpi-io/fence.c            \
     mpi-io/get_amode.c        \
     mpi-io/get_atom.c         \
     mpi-io/get_bytoff.c       \
diff --git a/src/mpi/romio/mpi-io/fence.c b/src/mpi/romio/mpi-io/fence.c
@@ -0,0 +1,71 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "mpioimpl.h"
+
+#ifdef HAVE_WEAK_SYMBOLS
+
+#if defined(HAVE_PRAGMA_WEAK)
+#pragma weak MPI_File_fence = PMPI_File_fence
+#elif defined(HAVE_PRAGMA_HP_SEC_DEF)
+#pragma _HP_SECONDARY_DEF PMPI_File_fence MPI_File_fence
+#elif defined(HAVE_PRAGMA_CRI_DUP)
+#pragma _CRI duplicate MPI_File_fence as PMPI_File_fence
+/* end of weak pragmas */
+#elif defined(HAVE_WEAK_ATTRIBUTE)
+int MPI_File_fence(MPI_File fh) __attribute__ ((weak, alias("PMPI_File_fence")));
+#endif
+
+/* Include mapping from MPI->PMPI */
+#define MPIO_BUILD_PROFILING
+#include "mpioprof.h"
+#endif
+
+/*@
+    MPI_File_fence - Causes all previous writes to become visible
+                     to other processes
+
+Input Parameters:
+. fh - file handle (handle)
+
+.N fortran
+@*/
+int MPI_File_fence(MPI_File fh)
+{
+    int error_code;
+    ADIO_File adio_fh;
+    static char myname[] = "MPI_FILE_SYNC";
+#ifdef MPI_hpux
+    int fl_xmpi;
+
+    HPMP_IO_START(fl_xmpi, BLKMPIFILESYNC, TRDTBLOCK, adio_fh, MPI_DATATYPE_NULL, -1);
+#endif /* MPI_hpux */
+    ROMIO_THREAD_CS_ENTER();
+
+    adio_fh = MPIO_File_resolve(fh);
+    /* --BEGIN ERROR HANDLING-- */
+    if ((adio_fh == NULL) || ((adio_fh)->cookie != ADIOI_FILE_COOKIE)) {
+        error_code = MPIO_Err_create_code(MPI_SUCCESS, MPIR_ERR_RECOVERABLE,
+                                          myname, __LINE__, MPI_ERR_ARG, "**iobadfh", 0);
+        error_code = MPIO_Err_return_file(MPI_FILE_NULL, error_code);
+        goto fn_exit;
+    }
+    MPIO_CHECK_WRITABLE(fh, myname, error_code);
+    /* --END ERROR HANDLING-- */
+
+    ADIO_Fence(adio_fh, &error_code);
+    /* --BEGIN ERROR HANDLING-- */
+    if (error_code != MPI_SUCCESS)
+        error_code = MPIO_Err_return_file(adio_fh, error_code);
+    /* --END ERROR HANDLING-- */
+
+#ifdef MPI_hpux
+    HPMP_IO_END(fl_xmpi, adio_fh, MPI_DATATYPE_NULL, -1);
+#endif /* MPI_hpux */
+
+  fn_exit:
+    ROMIO_THREAD_CS_EXIT();
+    return error_code;
+}
