add MPI_File_fence

autotools/buildme

@@ -0,0 +1,56 @@
+#!/bin/bash
+
+set -x
+
+installdir=`pwd`/install
+
+# add autotools install bin to our path
+export PATH=${installdir}/bin:$PATH
+
+# build autoconf
+if [ ! -f autoconf-2.69.tar.gz ] ; then
+  wget http://ftp.gnu.org/gnu/autoconf/autoconf-2.69.tar.gz
+fi
+rm -rf autoconf-2.69
+tar -zxf autoconf-2.69.tar.gz
+pushd autoconf-2.69
+  ./configure --prefix=$installdir
+  make
+  make install
+popd
+
+# build automake
+if [ ! -f automake-1.15.tar.gz ] ; then
+  wget http://ftp.gnu.org/gnu/automake/automake-1.15.tar.gz
+fi
+rm -rf automake-1.15
+tar -zxf automake-1.15.tar.gz
+pushd automake-1.15
+  ./configure --prefix=$installdir
+  make
+  make install
+popd
+
+# build libtool
+if [ ! -f libtool-2.4.6.tar.gz ] ; then
+  wget http://mirror.team-cymru.org/gnu/libtool/libtool-2.4.6.tar.gz
+fi
+rm -rf libtool-2.4.6
+tar -zxf libtool-2.4.6.tar.gz
+pushd libtool-2.4.6
+  ./configure --prefix=$installdir
+  make
+  make install
+popd
+
+# build pkg-config
+if [ ! -f pkg-config-0.27.1.tar.gz ] ; then
+  wget https://pkg-config.freedesktop.org/releases/pkg-config-0.27.1.tar.gz
+fi
+rm -rf pkg-config-0.27.1
+tar -zxf pkg-config-0.27.1.tar.gz
+pushd pkg-config-0.27.1
+    ./configure --prefix=$installdir
+    make
+    make install
+popd

buildme

@@ -0,0 +1,122 @@
+#!/bin/bash
+
+# run this to apply patches and create a new mv2 tarball
+
+set -x
+
+export PKG_CONFIG_PATH="/path/to/unifyfs/install/lib/pkgconfig":$PKG_CONFIG_PATH
+
+# build autotools that we need (one of the patches below changes configure)
+pushd autotools
+  ./buildme
+popd
+
+# add autotools install bin to our path
+export PATH=`pwd`/autotools/install/bin:$PATH
+
+# MPI installation directory
+installdir=`pwd`/install-gnu-main
+rm -rf $installdir
+
+# GNU compiler settings
+export CC=gcc 
+export CXX=g++ 
+export F77=gfortran
+export FC=gfortran
+#export MPICHLIB_CFLAGS="-g -O2"
+#export MPICHLIB_CXXFLAGS="-g -O2"
+#export MPICHLIB_FFLAGS="-g -O2 -fno-second-underscore"
+#export MPICHLIB_FCFLAGS="-g -O2 -fno-second-underscore"
+export CFLAGS="-I/usr/include/slurm"
+export MPICHLIB_CFLAGS="-g -O0 -I/usr/include/slurm"
+export MPICHLIB_CXXFLAGS="-g -O0"
+export MPICHLIB_FFLAGS="-g -O0 -fno-second-underscore"
+export MPICHLIB_FCFLAGS="-g -O0 -fno-second-underscore"
+
+# Intel compiler settings
+#export CC=icc 
+#export CXX=icpc
+#export F77=ifort
+#export FC=ifort
+#export MPICHLIB_CFLAGS="-g -O2"
+#export MPICHLIB_CXXFLAGS="-g -O2"
+#export MPICHLIB_FFLAGS="-g -O2"
+#export MPICHLIB_FCFLAGS="-g -O2"
+#export MPICHLIB_CFLAGS="-g -O0"
+#export MPICHLIB_CXXFLAGS="-g -O0"
+#export MPICHLIB_FFLAGS="-g -O0"
+#export MPICHLIB_FCFLAGS="-g -O0"
+
+# PGI compiler settings
+#export CC=pgcc 
+#export CXX=pgCC
+#export F77=pgf77
+##export FC=pgf90
+#export MPICHLIB_CFLAGS="-g -O2 -fPIC"
+#export MPICHLIB_CXXFLAGS="-g -O2 -fPIC"
+#export MPICHLIB_FFLAGS="-g -O2 -fPIC"
+#export MPICHLIB_FCFLAGS="-g -O2 -fPIC"
+
+# Pathscale compiler settings
+#export CC=pathcc 
+#export CXX=pathCC
+#export F77=pathf90
+##export FC=pathf90
+#export MPICHLIB_CFLAGS="-g -O2"
+#export MPICHLIB_CXXFLAGS="-g -O2"
+#export MPICHLIB_FFLAGS="-g -O2 -fno-second-underscore"
+#export MPICHLIB_FCFLAGS="-g -O2 -fno-second-underscore"
+
+#export CONFIG_FLAGS="--disable-fast --enable-g=dbg --enable-error-checking=runtime --enable-error-messages=all --enable-nmpi-as-mpi"
+export CONFIG_FLAGS="--enable-fast=all --enable-g=dbg --enable-nmpi-as-mpi"
+
+# PMI library:  PMGR_COLLECTIVE, SLURM
+export PMI="--with-pm=no --with-pmi=slurm"
+#export PMI="--with-pm=slurm --with-pmi=pmi2"
+#export PMI="--with-pm=no --with-pmi=pmi2"
+
+# Network devices:  PSM, Mellanox, Shared Memory, Mellanox + CUDA
+#export DEVICE_FLAGS="--with-device=ch3:psm --with-psm2"
+#export DEVICE_FLAGS="--with-device=ch3:psm --with-psm2 --with-psm2-lib=/usr/lib64"
+#export DEVICE_FLAGS="--with-device=ch3:psm --with-psm"
+#export DEVICE_FLAGS="--with-device=ch3:psm --with-psm --with-psm-lib=/usr/lib64"
+#export DEVICE_FLAGS="--with-device=ch3:psm"
+#export DEVICE_FLAGS="--with-device=ch3:mrail --with-rdma=gen2"
+#export DEVICE_FLAGS="--with-device=ch3:mrail --with-rdma=gen2 --enable-cuda --with-cuda=/opt/cudatoolkit-5.0"
+#export DEVICE_FLAGS="--with-device=ch3:nemesis"
+export DEVICE_FLAGS="--with-device=ch3"
+
+#export REG_CACHE="--disable-registration-cache"
+export REG_CACHE=""
+
+./autogen.sh
+
+rm -rf build
+mkdir build
+pushd build
+
+../configure \
+  --prefix=${installdir} \
+  --enable-f77 --enable-fc --enable-cxx \
+  ${CONFIG_FLAGS} \
+  --enable-shared --enable-sharedlibs=gcc \
+  --enable-debuginfo \
+  --disable-new-dtags \
+  ${PMI} \
+  ${DEVICE_FLAGS} \
+  ${REG_CACHE} \
+  --with-ch3-rank-bits=32 \
+  --enable-romio --with-file-system=lustre+nfs+ufs \
+  --disable-mpe --without-mpe 2>&1 > configure.out
+
+#  --enable-romio --with-file-system=gpfs+nfs+ufs \
+#  --enable-romio --with-file-system=lustre+nfs+ufs \
+#../configure \
+#--build=x86_64-redhat-linux-gnu --host=x86_64-redhat-linux-gnu --program-prefix= --disable-dependency-tracking --prefix=$installdir/mvapich2-x/gnu4.9.3/ifs/basic/slurm --exec-prefix=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm --bindir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/bin --sbindir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/sbin --sysconfdir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/etc --datadir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/share --includedir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/include --libdir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/lib64 --libexecdir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/libexec --localstatedir=/var --sharedstatedir=/var/lib --mandir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/share/man --infodir=${installdir}/mvapich2-x/gnu4.9.3/ifs/basic/slurm/share/info CC=gcc CXX=g++ F77=gfortran FC=gfortran --enable-ucr --disable-rpath --disable-static --enable-shared --disable-rdma-cm --without-hydra-ckpointlib --with-pm=slurm --with-pmi=pmi2 --with-device=ch3:psm --disable-mcast --enable-fast=O3 --enable-f77 -enable-fc --enable-cxx --enable-nmpi-as-mpi --enable-shared --enable-sharedlibs=gcc --disable-new-dtags --disable-registration-cache --with-ch3-rank-bits=32 --enable-romio --with-file-system=lustre+nfs+ufs --disable-mpe --without-mpe LDFLAGS=-Wl,-rpath,XORIGIN/placeholder 2>&1 > configure.out
+#exit 0
+
+#make clean
+
+#make -j8 VERBOSE=1 2>&1 > make.out
+
+make install VERBOSE=1

src/mpi/romio/adio/ad_gpfs/ad_gpfs.c

@@ -45,6 +45,7 @@ struct ADIOI_Fns_struct ADIO_GPFS_operations = {
     ADIOI_GEN_IreadStrided,     /* IreadStrided */
     ADIOI_GEN_IwriteStrided,    /* IwriteStrided */
     ADIOI_GPFS_Flush,   /* Flush */
+    ADIOI_GEN_Fence,    /* Fence */
     ADIOI_GEN_Resize,   /* Resize */
     ADIOI_GEN_Delete,   /* Delete */
     ADIOI_GEN_Feature,  /* Features */

src/mpi/romio/adio/ad_lustre/ad_lustre.c

@@ -32,6 +32,7 @@ struct ADIOI_Fns_struct ADIO_LUSTRE_operations = {
     ADIOI_GEN_IreadStrided,     /* IreadStrided */
     ADIOI_GEN_IwriteStrided,    /* IwriteStrided */
     ADIOI_GEN_Flush,    /* Flush */
+    ADIOI_GEN_Fence,    /* Fence */
     ADIOI_GEN_Resize,   /* Resize */
     ADIOI_GEN_Delete,   /* Delete */
     ADIOI_GEN_Feature,  /* Features */

src/mpi/romio/adio/ad_nfs/ad_nfs.c

@@ -32,6 +32,7 @@ struct ADIOI_Fns_struct ADIO_NFS_operations = {
     ADIOI_GEN_IreadStrided,     /* IreadStrided */
     ADIOI_GEN_IwriteStrided,    /* IwriteStrided */
     ADIOI_GEN_Flush,    /* Flush */
+    ADIOI_GEN_Fence,    /* Fence */
     ADIOI_NFS_Resize,   /* Resize */
     ADIOI_GEN_Delete,   /* Delete */
     ADIOI_NFS_Feature,  /* Features */

src/mpi/romio/adio/ad_testfs/Makefile.mk

@@ -20,6 +20,7 @@ romio_other_sources +=                \
     adio/ad_testfs/ad_testfs_iwrite.c \
     adio/ad_testfs/ad_testfs_wait.c   \
     adio/ad_testfs/ad_testfs_flush.c  \
+    adio/ad_testfs/ad_testfs_fence.c  \
     adio/ad_testfs/ad_testfs_seek.c   \
     adio/ad_testfs/ad_testfs_resize.c \
     adio/ad_testfs/ad_testfs_hints.c  \

src/mpi/romio/adio/ad_testfs/ad_testfs.c

@@ -30,6 +30,7 @@ struct ADIOI_Fns_struct ADIO_TESTFS_operations = {
     ADIOI_TESTFS_IreadStrided,  /* IreadStrided */
     ADIOI_TESTFS_IwriteStrided, /* IwriteStrided */
     ADIOI_TESTFS_Flush, /* Flush */
+    ADIOI_TESTFS_Fence, /* Fence */
     ADIOI_TESTFS_Resize,        /* Resize */
     ADIOI_TESTFS_Delete,        /* Delete */
     ADIOI_GEN_Feature,  /* Features */

src/mpi/romio/adio/ad_testfs/ad_testfs.h

@@ -62,6 +62,7 @@ void ADIOI_TESTFS_IwriteStrided(ADIO_File fd, const void *buf, int count,
                                 ADIO_Offset offset, ADIO_Request * request, int
                                 *error_code);
 void ADIOI_TESTFS_Flush(ADIO_File fd, int *error_code);
+void ADIOI_TESTFS_Fence(ADIO_File fd, int *error_code);
 void ADIOI_TESTFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
 ADIO_Offset ADIOI_TESTFS_SeekIndividual(ADIO_File fd, ADIO_Offset offset,
                                         int whence, int *error_code);

src/mpi/romio/adio/ad_testfs/ad_testfs_fence.c

@@ -0,0 +1,18 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "ad_testfs.h"
+#include "adioi.h"
+
+void ADIOI_TESTFS_Fence(ADIO_File fd, int *error_code)
+{
+    int myrank, nprocs;
+
+    *error_code = MPI_SUCCESS;
+
+    MPI_Comm_size(fd->comm, &nprocs);
+    MPI_Comm_rank(fd->comm, &myrank);
+    FPRINTF(stdout, "[%d/%d] ADIOI_TESTFS_Fence called on %s\n", myrank, nprocs, fd->filename);
+}

src/mpi/romio/adio/ad_ufs/ad_ufs.c

@@ -35,6 +35,7 @@ struct ADIOI_Fns_struct ADIO_UFS_operations = {
     ADIOI_GEN_IreadStrided,     /* IreadStrided */
     ADIOI_GEN_IwriteStrided,    /* IwriteStrided */
     ADIOI_GEN_Flush,    /* Flush */
+    ADIOI_GEN_Fence,    /* Fence */
     ADIOI_GEN_Resize,   /* Resize */
     ADIOI_GEN_Delete,   /* Delete */
     ADIOI_GEN_Feature,  /* Features */

src/mpi/romio/adio/ad_unify/Makefile.mk

@@ -15,6 +15,7 @@ romio_other_sources +=                          \
     adio/ad_unify/ad_unify_close.c \
     adio/ad_unify/ad_unify_io.c \
     adio/ad_unify/ad_unify_flush.c \
+    adio/ad_unify/ad_unify_fence.c \
     adio/ad_unify/ad_unify_delete.c \
     adio/ad_unify/ad_unify_fcntl.c \
     adio/ad_unify/ad_unify_resize.c \

src/mpi/romio/adio/ad_unify/ad_unify.c

@@ -35,6 +35,7 @@ struct ADIOI_Fns_struct ADIO_UNIFY_operations = {
     ADIOI_FAKE_IreadStrided,    /* IreadStrided */
     ADIOI_FAKE_IwriteStrided,   /* IwriteStrided */
     ADIOI_UNIFY_Flush,  /* Flush */
+    ADIOI_UNIFY_Fence,  /* Fence */
     ADIOI_UNIFY_Resize, /* Resize */
     ADIOI_UNIFY_Delete, /* Delete */
     ADIOI_UNIFY_Feature,

src/mpi/romio/adio/ad_unify/ad_unify_fence.c

@@ -0,0 +1,35 @@
+
+#include "ad_unify.h"
+#include "ad_unify_common.h"
+
+void ADIOI_UNIFY_Fence(ADIO_File fd, int *error_code)
+{
+    int ret;
+    *error_code = MPI_SUCCESS;
+    ADIOI_UNIFY_fs *unifyfs_blob = fd->fs_ptr;
+
+    /* Synchronize with file system to flush writes
+     * to make data visible to other processes. */
+    unifyfs_io_request sync_req = {
+        .op = UNIFYFS_IOREQ_OP_SYNC_META,
+        .gfid = unifyfs_blob->gfid,
+        .result.error = 0,
+    };
+    ret = unifyfs_dispatch_io(unifyfs_blob->fshdl, 1, &sync_req);
+    if (ret == UNIFYFS_SUCCESS) {
+        ret = unifyfs_wait_io(unifyfs_blob->fshdl, 1, &sync_req, 1);
+        if (ret == UNIFYFS_SUCCESS)
+            if (sync_req.result.error != 0) {
+                *error_code = MPIO_Err_create_code(MPI_SUCCESS,
+                                                   MPIR_ERR_RECOVERABLE, __func__, __LINE__,
+                                                   MPI_ERR_IO, "**io", "**io %s",
+                                                   strerror(sync_req.result.error));
+            }
+    }
+
+    /* Synchronize processes so that upon returning from fence,
+     * all procs know that all other procs have completed their flush. */
+    MPI_Barrier(fd->comm);
+
+    return;
+}

src/mpi/romio/adio/common/Makefile.mk

@@ -17,6 +17,7 @@ romio_other_sources +=                  \
     adio/common/ad_fcntl.c              \
     adio/common/ad_features.c           \
     adio/common/ad_flush.c              \
+    adio/common/ad_fence.c              \
     adio/common/ad_fstype.c             \
     adio/common/ad_get_sh_fp.c          \
     adio/common/ad_hints.c              \

src/mpi/romio/adio/common/ad_fence.c

@@ -0,0 +1,31 @@
+/*
+ * Copyright (C) by Argonne National Laboratory
+ *     See COPYRIGHT in top-level directory
+ */
+
+#include "adio.h"
+
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+void ADIOI_GEN_Fence(ADIO_File fd, int *error_code)
+{
+    int err;
+    static char myname[] = "ADIOI_GEN_FENCE";
+
+    /* For the general (common) case, we assume that the underlying
+     * file system implements a POSIX-compliant write,
+     * meaning that written data becomes visible to other processes
+     * immediately upon returning from a successful write call.
+     *
+     * For such file systems, there is no need to explicitly flush
+     * data or to synchronize with the file system to expose the
+     * data from those write calls. */
+
+    /* Even though the data flush step is a NOP,
+     * fence semantics still require that we synchronize processes. */
+    MPI_Barrier(fd->comm);
+
+    *error_code = MPI_SUCCESS;
+}

src/mpi/romio/adio/include/adio.h

@@ -396,6 +396,7 @@ void ADIO_IwriteStridedColl(ADIO_File fd, void *buf, int count,
 ADIO_Offset ADIO_SeekIndividual(ADIO_File fd, ADIO_Offset offset, int whence, int *error_code);
 void ADIO_Delete(char *filename, int *error_code);
 void ADIO_Flush(ADIO_File fd, int *error_code);
+void ADIO_Fence(ADIO_File fd, int *error_code);
 void ADIO_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
 void ADIO_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
 void ADIO_ResolveFileType(MPI_Comm comm, const char *filename, int *fstype,

src/mpi/romio/adio/include/adioi.h

@@ -226,6 +226,7 @@ struct ADIOI_Fns_struct {
                                      MPI_Datatype datatype, int file_ptr_type,
                                      ADIO_Offset offset, ADIO_Request * request, int *error_code);
     void (*ADIOI_xxx_Flush) (ADIO_File fd, int *error_code);
+    void (*ADIOI_xxx_Fence) (ADIO_File fd, int *error_code);
     void (*ADIOI_xxx_Resize) (ADIO_File fd, ADIO_Offset size, int *error_code);
     void (*ADIOI_xxx_Delete) (const char *filename, int *error_code);
     int (*ADIOI_xxx_Feature) (ADIO_File fd, int flag);
@@ -332,6 +333,8 @@ struct ADIOI_Fns_struct {
 
 #define ADIO_Flush(fd,error_code) (*(fd->fns->ADIOI_xxx_Flush))(fd,error_code)
 
+#define ADIO_Fence(fd,error_code) (*(fd->fns->ADIOI_xxx_Fence))(fd,error_code)
+
 #define ADIO_Resize(fd,size,error_code)                 \
     (*(fd->fns->ADIOI_xxx_Resize))(fd,size,error_code)
 
@@ -395,6 +398,7 @@ void ADIOI_Info_print_keyvals(MPI_Info info);
 
 void ADIOI_GEN_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t * fcntl_struct, int *error_code);
 void ADIOI_GEN_Flush(ADIO_File fd, int *error_code);
+void ADIOI_GEN_Fence(ADIO_File fd, int *error_code);
 void ADIOI_GEN_OpenColl(ADIO_File fd, int rank, int access_mode, int *error_code);
 void ADIOI_SCALEABLE_OpenColl(ADIO_File fd, int rank, int access_mode, int *error_code);
 void ADIOI_FAILSAFE_OpenColl(ADIO_File fd, int rank, int access_mode, int *error_code);

src/mpi/romio/include/mpio.h.in

@@ -246,6 +246,7 @@ int MPI_Register_datarep(const char *datarep, MPI_Datarep_conversion_function *r
 int MPI_File_set_atomicity(MPI_File fh, int flag) ROMIO_API_PUBLIC;
 int MPI_File_get_atomicity(MPI_File fh, int *flag) ROMIO_API_PUBLIC;
 int MPI_File_sync(MPI_File fh) ROMIO_API_PUBLIC;
+int MPI_File_fence(MPI_File fh) ROMIO_API_PUBLIC;
 
 /* Section 4.13.3 */
 #ifndef MPICH
@@ -549,6 +550,7 @@ int PMPI_Register_datarep(const char *,
 int PMPI_File_set_atomicity(MPI_File, int) ROMIO_API_PUBLIC;
 int PMPI_File_get_atomicity(MPI_File, int *) ROMIO_API_PUBLIC;
 int PMPI_File_sync(MPI_File) ROMIO_API_PUBLIC;
+int PMPI_File_fence(MPI_File) ROMIO_API_PUBLIC;
 
 /* Section 4.13.3 */
 #ifndef MPICH