|
| 1 | +#include <stdio.h> |
| 2 | +#include <limits.h> |
| 3 | +#include <stdlib.h> |
| 4 | +#include <libgen.h> |
| 5 | +#include <sys/types.h> |
| 6 | +#include <sys/stat.h> |
| 7 | +#include <mpi.h> |
| 8 | +#include <pio.h> |
| 9 | +#include "pio_config.h" |
| 10 | +#include <assert.h> |
| 11 | +#include <math.h> |
| 12 | +#include <string.h> |
| 13 | +#ifdef SPIO_ENABLE_GPTL_TIMING |
| 14 | +#include <gptl.h> |
| 15 | +#endif |
| 16 | +#if PIO_USE_NETCDF4 |
| 17 | +#include <netcdf.h> |
| 18 | +#endif |
| 19 | + |
| 20 | +#if PIO_USE_NETCDF4 |
| 21 | +#define ERR(rank, ret, err_msg) do{\ |
| 22 | + if(ret != NC_NOERR){\ |
| 23 | + printf("[%d]: FATAL ERROR : ", rank); \ |
| 24 | + printf(err_msg); \ |
| 25 | + printf(" (err = %d, line no = %d)\n", ret, __LINE__);\ |
| 26 | + return ret; \ |
| 27 | + }\ |
| 28 | +}while(0) |
| 29 | +#endif |
| 30 | + |
| 31 | +#define NDIMS 3 |
| 32 | +#define DIM_TIME_SZ 1 |
| 33 | +#define DIM_LEV_SZ 5 |
| 34 | +#define DIM_NCOLS_SZ_PER_PROC 10 |
| 35 | +#define FILE_PATH_MAX_SZ PATH_MAX+1 |
| 36 | + |
| 37 | +#if PIO_USE_NETCDF4 |
| 38 | + |
| 39 | +int get_nczarr_fname(MPI_Comm comm, int comm_rank, int comm_sz, |
| 40 | + const char *fname, size_t fname_sz, char *nczarr_fname, size_t nczarr_fname_sz) |
| 41 | +{ |
| 42 | + const char *nczarr_prefix = "file://"; |
| 43 | + const char *nczarr_suffix = "#mode=nczarr,file"; |
| 44 | + char real_fname[FILE_PATH_MAX_SZ + 1] = "\0"; |
| 45 | + |
| 46 | + assert(nczarr_fname_sz >= fname_sz); |
| 47 | + |
| 48 | + int real_fname_len = 0; |
| 49 | + if(comm_rank == 0){ |
| 50 | + char real_dname[FILE_PATH_MAX_SZ + 1]; |
| 51 | + char dname[FILE_PATH_MAX_SZ + 1]; |
| 52 | + char bname[FILE_PATH_MAX_SZ + 1]; |
| 53 | + |
| 54 | + strncpy(dname, fname, FILE_PATH_MAX_SZ); dname[FILE_PATH_MAX_SZ] = '\0'; |
| 55 | + strncpy(bname, fname, FILE_PATH_MAX_SZ); bname[FILE_PATH_MAX_SZ] = '\0'; |
| 56 | + |
| 57 | + char *dname_ptr = dirname(dname); |
| 58 | + char *bname_ptr = basename(bname); |
| 59 | + |
| 60 | + char *real_dname_ptr = realpath(dname_ptr, real_dname); |
| 61 | + ERR(comm_rank, (real_dname_ptr != NULL) ? 0 : -1, "Converting relative file path/name to full/real path failed"); |
| 62 | + |
| 63 | + snprintf(real_fname, FILE_PATH_MAX_SZ, "%s/%s", real_dname_ptr, bname_ptr); |
| 64 | + |
| 65 | + //int ret = mkdir(real_fname, S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IWGRP | S_IXGRP); |
| 66 | + //ERR(comm_rank, ((ret == -1) && (errno == EEXIST)) ? 0 : ret, "Creating directory, for NCZarr file, failed"); |
| 67 | + |
| 68 | + real_fname_len = strlen(real_fname) + 1; |
| 69 | + } |
| 70 | + |
| 71 | + MPI_Bcast(&real_fname_len, 1, MPI_INT, 0, comm); |
| 72 | + assert(((int )nczarr_fname_sz) >= real_fname_len); |
| 73 | + |
| 74 | + MPI_Bcast(real_fname, real_fname_len, MPI_CHAR, 0, comm); |
| 75 | + |
| 76 | + snprintf(nczarr_fname, nczarr_fname_sz, "%s%s%s", nczarr_prefix, real_fname, nczarr_suffix); |
| 77 | + |
| 78 | + return NC_NOERR; |
| 79 | +} |
| 80 | + |
| 81 | +int test_nc_file_ops(MPI_Comm comm, const char *fname, size_t fname_sz, int omode) |
| 82 | +{ |
| 83 | + int ret = NC_NOERR; |
| 84 | + int fh = -1; |
| 85 | + int comm_rank, comm_sz; |
| 86 | + const char *dim_names[] = {"time", "lev", "ncols"}; |
| 87 | + int dim_ids[NDIMS]; |
| 88 | + int varid = -1; |
| 89 | + |
| 90 | + MPI_Comm_rank(comm, &comm_rank); |
| 91 | + MPI_Comm_size(comm, &comm_sz); |
| 92 | + |
| 93 | + ret = nc_create_par(fname, omode, comm, MPI_INFO_NULL, &fh); ERR(comm_rank, ret, "Creating file failed"); |
| 94 | + |
| 95 | + size_t dim_sz[NDIMS] = {DIM_TIME_SZ, DIM_LEV_SZ, DIM_NCOLS_SZ_PER_PROC * comm_sz}; |
| 96 | + for(int i=0; i < NDIMS; i++){ |
| 97 | + ret = nc_def_dim(fh, dim_names[i], dim_sz[i], &dim_ids[i]); ERR(comm_rank, ret, "Defining dimension in file failed"); |
| 98 | + } |
| 99 | + |
| 100 | + double var[DIM_TIME_SZ][DIM_LEV_SZ][DIM_NCOLS_SZ_PER_PROC]; |
| 101 | + for(int i = 0; i < DIM_TIME_SZ; i++){ |
| 102 | + for(int j = 0; j < DIM_LEV_SZ; j++){ |
| 103 | + for(int k = 0; k < DIM_NCOLS_SZ_PER_PROC; k++){ |
| 104 | + var[i][j][k] = i * (DIM_LEV_SZ * DIM_NCOLS_SZ_PER_PROC * comm_sz) + j * (DIM_NCOLS_SZ_PER_PROC * comm_sz) + (DIM_NCOLS_SZ_PER_PROC * comm_rank) + k; |
| 105 | + } |
| 106 | + } |
| 107 | + } |
| 108 | + ret = nc_def_var(fh, "tmp_var", NC_DOUBLE, NDIMS, dim_ids, &varid); ERR(comm_rank, ret, "Defining variable (\"tmp_var\") failed"); |
| 109 | + |
| 110 | + ret = nc_enddef(fh); ERR(comm_rank, ret, "Ending define mode failed"); |
| 111 | + |
| 112 | + char info[] = "SCORPIO test"; |
| 113 | + ret = nc_put_att_text(fh, NC_GLOBAL, "info", strlen(info) + 1, info); ERR(comm_rank, ret, "Adding global attribute (\"info\" failed"); |
| 114 | + |
| 115 | + ret = nc_var_par_access(fh, varid, NC_COLLECTIVE); ERR(comm_rank, ret, "Setting parallel/collective access for variable failed"); |
| 116 | + |
| 117 | + size_t starts[NDIMS] = {0, 0, comm_rank * DIM_NCOLS_SZ_PER_PROC}; |
| 118 | + size_t counts[NDIMS] = {DIM_TIME_SZ, DIM_LEV_SZ, DIM_NCOLS_SZ_PER_PROC}; |
| 119 | + ret = nc_put_vara_double(fh, varid, starts, counts, (const double *)var); ERR(comm_rank, ret, "Writing variable in parallel failed"); |
| 120 | + |
| 121 | + ret = nc_close(fh); ERR(comm_rank, ret, "Closing file failed"); |
| 122 | + |
| 123 | + if(comm_rank == 0){ |
| 124 | + printf("Testing %s : SUCCESS\n", fname); fflush(stdout); |
| 125 | + } |
| 126 | + |
| 127 | + return ret; |
| 128 | +} |
| 129 | + |
| 130 | +int test_netcdf4p(MPI_Comm comm) |
| 131 | +{ |
| 132 | + int omode = NC_MPIIO | NC_CLOBBER | NC_NETCDF4; |
| 133 | + const char fname[] = "spio_test_netcdf4p.nc"; |
| 134 | + |
| 135 | + return test_nc_file_ops(comm, fname, strlen(fname) + 1, omode); |
| 136 | +} |
| 137 | + |
| 138 | +int test_netcdf4p_nczarr(MPI_Comm comm) |
| 139 | +{ |
| 140 | + int ret = NC_NOERR; |
| 141 | + int omode = NC_CLOBBER | NC_NETCDF4; |
| 142 | + const char fname[] = "./spio_test_netcdf4p_nczarr.file"; |
| 143 | + char nczarr_fname[FILE_PATH_MAX_SZ + 1] = "\0"; |
| 144 | + int comm_rank, comm_sz; |
| 145 | + |
| 146 | + MPI_Comm_rank(comm, &comm_rank); |
| 147 | + MPI_Comm_size(comm, &comm_sz); |
| 148 | + |
| 149 | + ret = get_nczarr_fname(comm, comm_rank, comm_sz, fname, strlen(fname) + 1, nczarr_fname, FILE_PATH_MAX_SZ + 1); |
| 150 | + ERR(comm_rank, ret, "Getting NCZarr file name failed"); |
| 151 | + |
| 152 | + return test_nc_file_ops(comm, nczarr_fname, strlen(nczarr_fname) + 1, omode); |
| 153 | +} |
| 154 | +#endif |
| 155 | + |
| 156 | +int main(int argc, char *argv[]) |
| 157 | +{ |
| 158 | + int ret = NC_NOERR; |
| 159 | + |
| 160 | +#if PIO_USE_NETCDF4 |
| 161 | +#ifdef SPIO_ENABLE_GPTL_TIMING |
| 162 | + GPTLinitialize(); |
| 163 | +#endif |
| 164 | + MPI_Init(&argc, &argv); |
| 165 | + |
| 166 | + ret = test_netcdf4p(MPI_COMM_WORLD); |
| 167 | + |
| 168 | + if(ret == NC_NOERR){ |
| 169 | + if((argc > 1) && (strcmp(argv[1], "--test-nczarr") == 0)){ |
| 170 | + ret = test_netcdf4p_nczarr(MPI_COMM_WORLD); |
| 171 | + } |
| 172 | + } |
| 173 | + |
| 174 | + MPI_Finalize(); |
| 175 | +#ifdef SPIO_ENABLE_GPTL_TIMING |
| 176 | + GPTLfinalize(); |
| 177 | +#endif |
| 178 | +#endif |
| 179 | + |
| 180 | + return ret; |
| 181 | +} |
0 commit comments