read large adjncy

Author: mnagaso

src/generate_databases/save_arrays_solver_hdf5.F90

@@ -137,7 +137,6 @@ subroutine save_arrays_solver_mesh_hdf5()
   integer(kind=8) :: sum_neighbors_adjncy, offset_neighbors_adjncy_this_proc
   integer(kind=8), dimension(0:NPROC-1) :: offset_neighbors_adjncy_i8
 
-
   ! saves mesh file external_mesh.h5
   tempstr = "/external_mesh.h5"
   filename = LOCAL_PATH(1:len_trim(LOCAL_PATH))//trim(tempstr)

src/shared/hdf5_manager.F90

@@ -194,6 +194,7 @@ module manager_hdf5
   interface h5_read_dataset_collect_hyperslab
     module procedure h5_read_dataset_1d_l_collect_hyperslab ! logical
     module procedure h5_read_dataset_1d_i_collect_hyperslab ! integer
+    module procedure h5_read_dataset_1d_i64_collect_hyperslab ! integer 64-bit
     module procedure h5_read_dataset_2d_i_collect_hyperslab
     module procedure h5_read_dataset_3d_i_collect_hyperslab
     module procedure h5_read_dataset_4d_i_collect_hyperslab
@@ -2605,7 +2606,60 @@ subroutine h5_read_dataset_1d_i_collect_hyperslab(dataset_name, data, offset_in,
     call h5_close_dataset()
   end subroutine h5_read_dataset_1d_i_collect_hyperslab
 
+  !
+!-------------------------------------------------------------------------------
 !
+
+  subroutine h5_read_dataset_1d_i64_collect_hyperslab(dataset_name, data, offset_in, if_collective)
+    implicit none
+    character(len=*), intent(in)                                :: dataset_name
+    integer, dimension(:), intent(inout), target                :: data
+    integer(kind=8), dimension(:), intent(in)                           :: offset_in
+    logical, intent(in)                                         :: if_collective
+    ! local parameters
+    integer                                                     :: rank = 1
+    integer(HSIZE_T), dimension(1)                              :: dim
+    integer(HSIZE_T), dimension(1)                              :: count ! size of hyperslab
+    integer(HSSIZE_T), dimension(1)                             :: offset ! the position where the datablock is inserted
+
+    dim = shape(data)
+    offset = offset_in ! convert data type
+
+    ! open dataset
+    call h5_open_dataset2(trim(dataset_name))
+
+    ! select a place where data is inserted.
+    count(1)  = dim(1)
+
+    ! select hyperslab in the file
+    call h5screate_simple_f(rank,count, mem_dspace_id, error)
+    call check_error()
+    call h5dget_space_f(dataset_id, file_dspace_id, error)
+    call check_error()
+    call h5sselect_hyperslab_f(file_dspace_id, H5S_SELECT_SET_F, offset, count, error)
+    call check_error()
+    call h5_create_dataset_prop_list(if_collective)
+
+    call h5_check_arr_dim(dim)
+
+    ! write array using Fortran pointer
+    !call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, data, dim, error, &
+    !                file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id)
+    ! use F2003 API
+    f_ptr = c_loc(data(1))
+    call h5dread_f(dataset_id, H5T_NATIVE_INTEGER, f_ptr, error, &
+                    file_space_id=file_dspace_id, mem_space_id=mem_dspace_id, xfer_prp=plist_id)
+    if (error /= 0) write(*,*) 'hdf5 dataset write failed for ', dataset_name
+    call check_error()
+    call h5_close_prop_list(dataset_name)
+    call h5sclose_f(mem_dspace_id, error)
+    call check_error()
+    call h5sclose_f(file_dspace_id, error)
+    call check_error()
+    call h5_close_dataset()
+  end subroutine h5_read_dataset_1d_i64_collect_hyperslab
+
+
 !-------------------------------------------------------------------------------
 !
 

src/specfem3D/read_mesh_databases_hdf5.F90

@@ -158,6 +158,9 @@ subroutine read_mesh_databases_hdf5()
   integer, dimension(0:NPROC-1) :: offset_neighbors_xadj
   integer, dimension(0:NPROC-1) :: offset_neighbors_adjncy
 
+  ! avoid integer overflow
+  integer(kind=8) :: sum_offset_neighbors_adjncy_this_proc
+  integer(kind=8), dimension(0:NPROC-1) :: offset_neighbors_adjncy_i8
 
   ! user output
   if (myrank == 0) then
@@ -203,6 +206,8 @@ subroutine read_mesh_databases_hdf5()
     ! mesh adjacency
     call h5_read_dataset_collect_hyperslab("offset_neighbors_xadj",offset_neighbors_xadj, (/0/), H5_COL)
     call h5_read_dataset_collect_hyperslab("offset_neighbors_adjncy",offset_neighbors_adjncy, (/0/), H5_COL)
+    ! convert to 8-byte integers
+    offset_neighbors_adjncy_i8 = offset_neighbors_adjncy
 
     ! info about external mesh simulation
     dsetname = "nspec"
@@ -1631,8 +1636,9 @@ subroutine read_mesh_databases_hdf5()
     call h5_read_dataset_collect_hyperslab(dsetname, neighbors_xadj, &
             (/sum(offset_neighbors_xadj(0:myrank-1))/), H5_COL)
     dsetname = "neighbors_adjncy"
+    sum_offset_neighbors_adjncy_this_proc = sum(offset_neighbors_adjncy_i8(0:myrank-1))
     call h5_read_dataset_collect_hyperslab(dsetname, neighbors_adjncy, &
-            (/sum(offset_neighbors_adjncy(0:myrank-1))/), H5_COL)
+            (/sum_offset_neighbors_adjncy_this_proc/), H5_COL)
   endif
   call bcast_all_i_for_database(neighbors_xadj(1), size(neighbors_xadj))
   call bcast_all_i_for_database(neighbors_adjncy(1), size(neighbors_adjncy))