Skip to content

Commit 13e6d43

Browse files
anderspandersp
authored
Merge pull request #195 from geodynamics/developer
Merge developer to master
2 parents 77a0d09 + 260d714 commit 13e6d43

File tree

10 files changed

+276
-167
lines changed

10 files changed

+276
-167
lines changed

‎configs/Dockerfile‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
FROM ubuntu:20.04
2+
3+
WORKDIR /home
4+
5+
# build environment
6+
RUN apt-get update && \
7+
DEBIAN_FRONTEND='noninteractive' \
8+
DEBCONF_NONINTERACTIVE_SEEN='true' \
9+
apt-get install --yes wget build-essential git cmake gfortran openmpi-bin && \
10+
\
11+
\
12+
# install hdf5 1.12.1, and remain packages \
13+
wget https://support.hdfgroup.org/ftp/HDF5/releases/hdf5-1.12/hdf5-1.12.1/src/hdf5-1.12.1.tar.gz && \
14+
tar xzf hdf5-1.12.1.tar.gz && rm hdf5-1.12.1.tar.gz && cd hdf5-1.12.1 && ./configure --prefix=/usr/local/hdf5 --enable-parallel --enable-build-mode=production && \
15+
make -j4 && make install && cd .. && \
16+
apt-get install --yes libproj-dev libhdf5-mpi-dev libfftw3-dev libfftw3-mpi-dev \
17+
liblapack-dev python3-h5py libcurl3-dev && \
18+
\
19+
\
20+
# install zfp and H5Z-ZFP \
21+
git -c http.sslVerify=false clone https://github.com/LLNL/zfp.git && cd zfp/ && \
22+
sed -i 's/# DEFS += -DBIT_STREAM_WORD_TYPE=uint8/DEFS += -DBIT_STREAM_WORD_TYPE=uint8/g' Config && \
23+
make && cd .. && \
24+
git -c http.sslVerify=false clone https://github.com/LLNL/H5Z-ZFP.git && cd H5Z-ZFP/ && \
25+
make FC= CC=mpicc ZFP_HOME=/home/zfp HDF5_HOME=/usr/local/hdf5 all install && \
26+
cd ..
27+
28+
# changing BUILD_TIME will break the cache here so that sw4 can be built each time
29+
ARG BUILD_TIME=unknown
30+
31+
# build sw4
32+
RUN git -c http.sslVerify=false clone https://github.com/geodynamics/sw4.git && \
33+
cd sw4/ && git switch developer && rm -r .git/ && \
34+
make sw4 CXX=mpicxx FC=gfortran debug=no proj=yes hdf5=yes fftw=yes zfp=yes prec=double EXTRA_LINK_FLAGS="-L/usr/lib64 -lgfortran -lhdf5 -llapack" HDF5ROOT=/usr/local/hdf5 FFTWHOME=/usr/lib/x86_64-linux-gnu ZFPROOT=/home/zfp H5ZROOT=/home/H5Z-ZFP/install -j4
35+
36+
ENV PATH="$PATH:/home/sw4/optimize_mp" \
37+
LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/hdf5/lib"
38+
39+
# run sw4
40+
# I typically use:
41+
# docker run --cap-add=SYS_PTRACE -it -v $(pwd):/home/model/ sw4:zfp
42+
# to run the docker container in avoid of the CMA issue (https://github.com/open-mpi/ompi/issues/4948)

‎src/ESSI3D.C‎

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -313,6 +313,7 @@ void ESSI3D::open_vel_file(int a_cycle, std::string& a_path, float_sw4 a_time,
313313
Sarray& a_Z) {
314314
bool debug = false;
315315
/* debug = true; */
316+
MPI_Comm comm = mEW->m_cartesian_communicator;
316317

317318
bool is_root = true;
318319
int g = mEW->mNumberOfGrids - 1;
@@ -352,21 +353,13 @@ void ESSI3D::open_vel_file(int a_cycle, std::string& a_path, float_sw4 a_time,
352353
if (!m_isRestart) {
353354
m_hdf5helper->write_header(h, lonlat_origin, az, origin, a_cycle, a_time,
354355
dt);
355-
356-
// Write z coodinates if necesito
357-
if (mEW->topographyExists()) {
358-
compute_image(a_Z, 0, 0);
359-
if (m_precision == 4)
360-
m_hdf5helper->write_topo(m_floatField[0]);
361-
else if (m_precision == 8)
362-
m_hdf5helper->write_topo(m_doubleField[0]);
363-
}
364356
}
365-
366357
if (debug)
367358
cout << "Creating hdf5 velocity fields..." << endl;
368359
}
369360

361+
MPI_Barrier(comm);
362+
370363
if (m_dumpInterval > 0) {
371364
int nstep = (int)ceil(m_ntimestep / m_dumpInterval);
372365
if (m_compressionMode > 0)
@@ -384,16 +377,27 @@ void ESSI3D::open_vel_file(int a_cycle, std::string& a_path, float_sw4 a_time,
384377
m_bufferInterval);
385378
}
386379

387-
MPI_Comm comm = mEW->m_cartesian_communicator;
388380
MPI_Barrier(comm);
389381

390382
is_root = false;
391383
m_hdf5helper->create_file(true, is_root);
392384

393-
m_hdf5_time += (MPI_Wtime() - hdf5_time);
394-
395385
m_nbufstep = 0;
396386
m_fileOpen = true;
387+
388+
if (!m_isRestart) {
389+
// Write z coodinates if has topo
390+
if (mEW->topographyExists()) {
391+
compute_image(a_Z, 0, 0);
392+
if (m_precision == 4)
393+
m_hdf5helper->write_topo(m_floatField[0]);
394+
else if (m_precision == 8)
395+
m_hdf5helper->write_topo(m_doubleField[0]);
396+
}
397+
}
398+
399+
m_hdf5_time += (MPI_Wtime() - hdf5_time);
400+
397401
return;
398402
}
399403

@@ -419,9 +423,10 @@ void ESSI3D::write_image_hdf5(int cycle, std::string& path, float_sw4 t,
419423
/* debug = true; */
420424

421425
for (int i = 0; i < 3; i++) {
426+
int nstep = (int)floor(m_ntimestep / m_dumpInterval);
422427
compute_image(a_U[g], i, cycle);
423428
if (cycle > 0 &&
424-
(m_nbufstep == m_bufferInterval - 1 || cycle == m_ntimestep)) {
429+
(m_nbufstep == m_bufferInterval - 1 || cycle == nstep)) {
425430
if (m_precision == 4)
426431
m_hdf5helper->write_vel((void*)m_floatField[i], i, cycle,
427432
m_nbufstep + 1);

‎src/ESSI3DHDF5.C‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -314,6 +314,8 @@ void ESSI3DHDF5::write_topo(void* window_array) {
314314
ierr = H5Sclose(dataspace_id);
315315
ierr = H5Dclose(dataset_id);
316316

317+
H5Fflush(m_file_id, H5F_SCOPE_LOCAL);
318+
317319
if (debug && (myRank == 0))
318320
cerr << "Done writing hdf5 z coordinate: " << m_filename << endl;
319321
#endif

‎src/EW.C‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7238,7 +7238,6 @@ void EW::extractTopographyFromGMG( std::string a_topoFileName )
72387238
start_time = MPI_Wtime();
72397239
#ifdef USE_HDF5
72407240
int verbose = mVerbose;
7241-
std::string rname ="EW::extractTopographyFromGMG";
72427241
Sarray gridElev;
72437242
herr_t ierr;
72447243
hid_t file_id, dataset_id, datatype_id, group_id, dataspace_id;
@@ -8495,6 +8494,9 @@ void EW::set_to_zero_at_receiver( vector<Sarray> & a_U,
84958494
#pragma omp parallel for
84968495
for( int s=0 ; s < time_series.size() ; s++ )
84978496
{
8497+
if (!time_series[s]->myPoint())
8498+
continue;
8499+
84988500
int g = time_series[s]->m_grid0;
84998501
int i0= time_series[s]->m_i0;
85008502
int j0= time_series[s]->m_j0;

‎src/Image.C‎

Lines changed: 26 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -1347,25 +1347,28 @@ void Image::writeImagePlane_2(int cycle, std::string &path, float_sw4 t )
13471347
H5Sclose(dset_space);
13481348
H5Dclose(dset);
13491349

1350-
if( mGridinfo == 1 )
1350+
char grid_name[16];
1351+
if( mGridinfo >= 1 )
13511352
{
1352-
int g=mEW->mNumberOfGrids-1;
1353-
int globalSizes[3] = {mEW->m_global_nx[g], mEW->m_global_ny[g], mEW->m_global_nz[g]} ;
1354-
if(mLocationType == Image::X)
1355-
globalSizes[0] = 1;
1356-
if (mLocationType == Image::Y)
1357-
globalSizes[1] = 1;
1358-
if (mLocationType == Image::Z)
1359-
globalSizes[2] = 1;
1360-
1361-
dims = globalSizes[0]*globalSizes[1]*globalSizes[2];
1362-
dset_space = H5Screate_simple(1, &dims, NULL);
1363-
/* cout << "Rank " << mEW->getRank() << " creating grid array with length " << dims << endl; */
1364-
dset = H5Dcreate(h5_fid, "grid", dtype, dset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
1365-
if( dset < 0 )
1366-
cout << "ERROR: Image::writeImagePlane_2 could not create HDF5 grid dataset" << endl;
1367-
H5Sclose(dset_space);
1368-
H5Dclose(dset);
1353+
for (int g = mEW->mNumberOfCartesianGrids; g < mEW->mNumberOfGrids; g++) {
1354+
int globalSizes[3] = {mEW->m_global_nx[g], mEW->m_global_ny[g], mEW->m_global_nz[g]} ;
1355+
if(mLocationType == Image::X)
1356+
globalSizes[0] = 1;
1357+
if (mLocationType == Image::Y)
1358+
globalSizes[1] = 1;
1359+
if (mLocationType == Image::Z)
1360+
globalSizes[2] = 1;
1361+
1362+
dims = globalSizes[0]*globalSizes[1]*globalSizes[2];
1363+
dset_space = H5Screate_simple(1, &dims, NULL);
1364+
/* cout << "Rank " << mEW->getRank() << " creating grid array with length " << dims << endl; */
1365+
sprintf(grid_name, "grid%d", g);
1366+
dset = H5Dcreate(h5_fid, grid_name, dtype, dset_space, H5P_DEFAULT, H5P_DEFAULT, H5P_DEFAULT);
1367+
if( dset < 0 )
1368+
cout << "ERROR: Image::writeImagePlane_2 could not create HDF5 grid dataset" << endl;
1369+
H5Sclose(dset_space);
1370+
H5Dclose(dset);
1371+
}
13691372
}
13701373

13711374
delete [] grid_size;
@@ -1481,7 +1484,7 @@ void Image::writeImagePlane_2(int cycle, std::string &path, float_sw4 t )
14811484
}
14821485
} // End if ihavearray
14831486

1484-
if( mGridinfo == 1 )
1487+
if( mGridinfo >= 1 )
14851488
add_grid_to_file_hdf5( s.str().c_str(), iwrite, 0);
14861489
/* if( mGridinfo == 2 ) */
14871490
/* add_grid_filenames_to_file( s.str().c_str() ); */
@@ -1495,6 +1498,7 @@ void Image::writeImagePlane_2(int cycle, std::string &path, float_sw4 t )
14951498
void Image::add_grid_to_file_hdf5( const char* fname, bool iwrite, size_t offset )
14961499
{
14971500
bool ihavearray = plane_in_proc(m_gridPtIndex[0]);
1501+
char grid_name[16];
14981502
if( ihavearray )
14991503
{
15001504
#ifdef USE_HDF5
@@ -1512,17 +1516,16 @@ void Image::add_grid_to_file_hdf5( const char* fname, bool iwrite, size_t offset
15121516
if( mEW->usingParallelFS() )
15131517
MPI_Barrier( m_mpiComm_writers );
15141518

1519+
sprintf(grid_name, "grid%d", g);
15151520
if( m_double )
15161521
{
15171522
char dblStr[]="double";
1518-
m_pio[g]->write_array_hdf5(fname, NULL, "grid", 1, m_gridimage->m_doubleField[g], offset, dblStr );
1519-
offset += (globalSizes[0]*globalSizes[1]*globalSizes[2]*sizeof(double));
1523+
m_pio[g]->write_array_hdf5(fname, NULL, grid_name, 1, m_gridimage->m_doubleField[g], 0, dblStr );
15201524
}
15211525
else
15221526
{
15231527
char fltStr[]="float";
1524-
m_pio[g]->write_array_hdf5(fname, NULL, "grid", 1, m_gridimage->m_floatField[g], offset, fltStr );
1525-
offset += (globalSizes[0]*globalSizes[1]*globalSizes[2]*sizeof(float));
1528+
m_pio[g]->write_array_hdf5(fname, NULL, grid_name, 1, m_gridimage->m_floatField[g], 0, fltStr );
15261529
}
15271530
}
15281531
#endif

‎src/Qspline.C‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -193,7 +193,11 @@ Qspline::Qspline( int npts, float_sw4* fun, float_sw4 tmin, float_sw4 dt, int bc
193193
delete[] ipiv;
194194

195195
m_npts = npts;
196-
m_polcof = new float_sw4[6*(npts-1)];
196+
if (npts > 1)
197+
m_polcof = new float_sw4[6*(npts-1)];
198+
else
199+
m_polcof = new float_sw4[6];
200+
197201
for( int i= 0 ; i < npts-1 ; i++ )
198202
{
199203
m_polcof[ 6*i] = fun[i];

‎src/SfileOutput.C‎

Lines changed: 28 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -351,8 +351,22 @@ void SfileOutput::compute_image( vector<Sarray>& a_U, vector<Sarray>& a_Rho,
351351
{
352352
int stH = mSampleH;
353353
int stV = mSampleV;
354+
vector<int> hh(mEW->mNumberOfGrids, 1);
354355
double my_z, up_z, down_z, up_v, down_v;
355356

357+
// Calculate horizontal factor of each grid for topo data access
358+
for (int i = mEW->mNumberOfGrids-2; i >= 0; i--) {
359+
// No h increase from Cartesian to Curvilinear
360+
if (i == mEW->mNumberOfCartesianGrids-1)
361+
hh[i] = hh[i+1];
362+
else
363+
hh[i] = hh[i+1] * 2;
364+
}
365+
366+
// debug
367+
/* int myRank; */
368+
/* MPI_Comm_rank( mEW->m_1d_communicator, &myRank); */
369+
356370
/* if (mMode== RHO && m_parallel_io[0]->proc_zero()) { */
357371
/* for( int g=mEW->mNumberOfCartesianGrids ; g < mEW->mNumberOfGrids ; g++ ) { */
358372
/* cout << "g="<< g << ", a_Z.m_kb="<< a_Z[g].m_kb << ", a_Z.m_ke=" << a_Z[g].m_ke << ", gz=" << mWindow[g][5] << endl; */
@@ -390,15 +404,15 @@ void SfileOutput::compute_image( vector<Sarray>& a_U, vector<Sarray>& a_Rho,
390404

391405
if( mMode == RHO || mMode == QP || mMode == QS ) { // these modes just copy the values straight from the array
392406
if( m_double ) {
393-
#pragma omp parallel for
407+
/* #pragma omp parallel for */
394408
for( int k=mWindow[g][4] ; k <= mWindow[g][5] ; k+=stV )
395409
for( int j=mWindow[g][2] ; j <= mWindow[g][3] ; j+=stH )
396410
for( int i=mWindow[g][0] ; i <= mWindow[g][1] ; i+=stH ) {
397411
size_t ind = (k-mWindow[g][4])/stV+nkw*(j-mWindow[g][2])/stH+njkw*(i-mWindow[g][0])/stH;
398412
if (g < mEW->mNumberOfCartesianGrids)
399413
m_doubleField[g][ind] = (double)((*data1)(1,i,j,k));
400414
else {
401-
double z_kl = -mEW->mTopo(i,j,1);
415+
double z_kl = a_Z[gz](i,j,kl);
402416
double z_ku = a_Z[gz](i,j,ku);
403417
my_z = z_kl + (z_ku - z_kl)*(k-1)/(double)(ku-1);
404418
int t = 1;
@@ -415,16 +429,15 @@ void SfileOutput::compute_image( vector<Sarray>& a_U, vector<Sarray>& a_Rho,
415429
}
416430
}
417431
else {
418-
#pragma omp parallel for
432+
/* #pragma omp parallel for */
419433
for( int k=mWindow[g][4] ; k <= mWindow[g][5] ; k+=stV )
420434
for( int j=mWindow[g][2] ; j <= mWindow[g][3] ; j+=stH )
421435
for( int i=mWindow[g][0] ; i <= mWindow[g][1] ; i+=stH ) {
422436
size_t ind = (k-mWindow[g][4])/stV+nkw*(j-mWindow[g][2])/stH+njkw*(i-mWindow[g][0])/stH;
423437
if (g < mEW->mNumberOfCartesianGrids)
424438
m_floatField[g][ind] = (float)((*data1)(1,i,j,k));
425439
else {
426-
// debug
427-
double z_kl = -mEW->mTopo(i,j,1);
440+
double z_kl = a_Z[gz](i,j,kl);
428441
double z_ku = a_Z[gz](i,j,ku);
429442
my_z = z_kl + (z_ku - z_kl)*(k-1)/(double)(ku-1);
430443
int t = 1;
@@ -443,15 +456,15 @@ void SfileOutput::compute_image( vector<Sarray>& a_U, vector<Sarray>& a_Rho,
443456
}
444457
else if( mMode == P ) {
445458
if( m_double ) {
446-
#pragma omp parallel for
459+
/* #pragma omp parallel for */
447460
for( int k=mWindow[g][4] ; k <= mWindow[g][5] ; k+=stV )
448461
for( int j=mWindow[g][2] ; j <= mWindow[g][3] ; j+=stH )
449462
for( int i=mWindow[g][0] ; i <= mWindow[g][1] ; i+=stH ) {
450463
size_t ind = (k-mWindow[g][4])/stV+nkw*(j-mWindow[g][2])/stH+njkw*(i-mWindow[g][0])/stH;
451464
if (g < mEW->mNumberOfCartesianGrids)
452465
m_doubleField[g][ind] = (double)sqrt((2*((*data2)(1,i,j,k)) +((*data3)(1,i,j,k)))/((*data1)(1,i,j,k)));
453466
else {
454-
double z_kl = -mEW->mTopo(i,j,1);
467+
double z_kl = a_Z[gz](i,j,kl);
455468
double z_ku = a_Z[gz](i,j,ku);
456469
my_z = z_kl + (z_ku - z_kl)*(k-1)/(double)(ku-1);
457470
int t = 1;
@@ -477,15 +490,15 @@ void SfileOutput::compute_image( vector<Sarray>& a_U, vector<Sarray>& a_Rho,
477490
}
478491
}
479492
else {
480-
#pragma omp parallel for
493+
/* #pragma omp parallel for */
481494
for( int k=mWindow[g][4] ; k <= mWindow[g][5] ; k+=stV )
482495
for( int j=mWindow[g][2] ; j <= mWindow[g][3] ; j+=stH )
483496
for( int i=mWindow[g][0] ; i <= mWindow[g][1] ; i+=stH ) {
484497
size_t ind = (k-mWindow[g][4])/stV+nkw*(j-mWindow[g][2])/stH+njkw*(i-mWindow[g][0])/stH;
485498
if (g < mEW->mNumberOfCartesianGrids)
486499
m_floatField[g][ind] = (float)sqrt((2.0*((*data2)(1,i,j,k)) +((*data3)(1,i,j,k)))/((*data1)(1,i,j,k)));
487500
else {
488-
double z_kl = -mEW->mTopo(i,j,1);
501+
double z_kl = a_Z[gz](i,j,kl);
489502
double z_ku = a_Z[gz](i,j,ku);
490503
my_z = z_kl + (z_ku - z_kl)*(k-1)/(double)(ku-1);
491504
int t = 1;
@@ -520,15 +533,15 @@ void SfileOutput::compute_image( vector<Sarray>& a_U, vector<Sarray>& a_Rho,
520533
}
521534
else if( mMode == S ) {
522535
if( m_double ) {
523-
#pragma omp parallel for
536+
/* #pragma omp parallel for */
524537
for( int k=mWindow[g][4] ; k <= mWindow[g][5] ; k+=stV )
525538
for( int j=mWindow[g][2] ; j <= mWindow[g][3] ; j+=stH )
526539
for( int i=mWindow[g][0] ; i <= mWindow[g][1] ; i+=stH ) {
527540
size_t ind = (k-mWindow[g][4])/stV+nkw*(j-mWindow[g][2])/stH+njkw*(i-mWindow[g][0])/stH;
528541
if (g < mEW->mNumberOfCartesianGrids)
529542
m_doubleField[g][ind] = (double)sqrt(((*data2)(1,i,j,k))/((*data1)(1,i,j,k)));
530543
else {
531-
double z_kl = -mEW->mTopo(i,j,1);
544+
double z_kl = a_Z[gz](i,j,kl);
532545
double z_ku = a_Z[gz](i,j,ku);
533546
my_z = z_kl + (z_ku - z_kl)*(k-1)/(double)(ku-1);
534547
int t = 1;
@@ -553,15 +566,15 @@ void SfileOutput::compute_image( vector<Sarray>& a_U, vector<Sarray>& a_Rho,
553566
}
554567
}
555568
else {
556-
#pragma omp parallel for
569+
/* #pragma omp parallel for */
557570
for( int k=mWindow[g][4] ; k <= mWindow[g][5] ; k+=stV )
558571
for( int j=mWindow[g][2] ; j <= mWindow[g][3] ; j+=stH )
559572
for( int i=mWindow[g][0] ; i <= mWindow[g][1] ; i+=stH ) {
560573
size_t ind = (k-mWindow[g][4])/stV+nkw*(j-mWindow[g][2])/stH+njkw*(i-mWindow[g][0])/stH;
561574
if (g < mEW->mNumberOfCartesianGrids)
562575
m_floatField[g][ind] = (float)sqrt(((*data2)(1,i,j,k))/((*data1)(1,i,j,k)));
563576
else {
564-
double z_kl = -mEW->mTopo(i,j,1);
577+
double z_kl = a_Z[gz](i,j,kl);
565578
double z_ku = a_Z[gz](i,j,ku);
566579
my_z = z_kl + (z_ku - z_kl)*(k-1)/(double)(ku-1);
567580
int t = 1;
@@ -627,7 +640,7 @@ void SfileOutput::compute_image( vector<Sarray>& a_U, vector<Sarray>& a_Rho,
627640
}
628641
}
629642
}
630-
}
643+
}// End for g < mEW->mNumberOfGrids
631644
}
632645

633646
//-----------------------------------------------------------------------
@@ -894,7 +907,7 @@ void SfileOutput::write_image(const char *fname, std::vector<Sarray>& a_Z )
894907
int nj = (int)(mWindow[real_g][3]-mWindow[real_g][2])/stH+1;
895908
int nk = mWindow[real_g][5];
896909
float* zfp = new float[npts];
897-
#pragma omp parallel for
910+
/* #pragma omp parallel for */
898911
for( int j=mWindow[real_g][2] ; j <= mWindow[real_g][3] ; j+=stH )
899912
for( int i=mWindow[real_g][0] ; i <= mWindow[real_g][1] ; i+=stH ) {
900913
size_t ind = (size_t)(j-mWindow[real_g][2])/stH+nj*(i-mWindow[real_g][0])/stH;

0 commit comments

Comments
 (0)