Merge pull request #1815 from danielpeter/devel

updates azure tests; updates Mesh_Par_file reading (optional parameters); updates rhostore storage & usage

Author: danielpeter

.azure-pipelines.yml

@@ -84,7 +84,7 @@ jobs:
     displayName: 'Run checks'
 
 - job: compilation_default
-  # ubuntu-latest: ubuntu-22.04 w/ GCC 11
+  # ubuntu-latest: ubuntu-24.04 w/ GCC 13.3
   displayName: 'Compilation Default GCC'
   steps:
   - template: .azure-pipelines/install-template.yml
@@ -96,14 +96,14 @@ jobs:
       CUDA: false
       BUILD: true
 
-- job: compilation_default_gcc9
-  displayName: 'Compilation Default GCC 9'
+- job: compilation_default_gcc10
+  displayName: 'Compilation Default GCC 10'
   pool:
-    vmImage: 'ubuntu-20.04'
+    vmImage: 'ubuntu-22.04'
   variables:
-    CC: gcc-9
-    CXX: g++-9
-    FC: gfortran-9
+    CC: gcc-10
+    CXX: g++-10
+    FC: gfortran-10
   steps:
   - template: .azure-pipelines/install-template.yml
     parameters:
@@ -114,14 +114,14 @@ jobs:
       CUDA: false
       BUILD: true
 
-- job: compilation_default_gcc10
-  displayName: 'Compilation Default GCC 10'
+- job: compilation_default_gcc11
+  displayName: 'Compilation Default GCC 11'
   pool:
-    vmImage: 'ubuntu-20.04'
+    vmImage: 'ubuntu-22.04'
   variables:
-    CC: gcc-10
-    CXX: g++-10
-    FC: gfortran-10
+    CC: gcc-11
+    CXX: g++-11
+    FC: gfortran-11
   steps:
   - template: .azure-pipelines/install-template.yml
     parameters:
@@ -132,52 +132,52 @@ jobs:
       CUDA: false
       BUILD: true
 
-- job: compilation_CUDA11_gcc9
-  displayName: 'Compilation CUDA 11 GCC 9'
+- job: compilation_CUDA11_gcc10
+  displayName: 'Compilation CUDA 11 GCC 10'
   pool:
-    vmImage: 'ubuntu-20.04'
+    vmImage: 'ubuntu-22.04'
   variables:
-    CC: gcc-9
-    CXX: g++-9
-    FC: gfortran-9
+    CC: gcc-10
+    CXX: g++-10
+    FC: gfortran-10
   steps:
   - template: .azure-pipelines/install-template.yml
     parameters:
       CUDA: true
-      CUDA_V: '11.4'
+      CUDA_V: '11.7'
   - template: .azure-pipelines/configure-template.yml
     parameters:
-      TESTFLAGS: '--with-mpi --enable-vectorization --with-cuda=cuda10'
+      TESTFLAGS: '--with-mpi --enable-vectorization --with-cuda=cuda11'
       CUDA: true
       BUILD: true
 
-- job: compilation_CUDA11_gcc10
-  displayName: 'Compilation CUDA 11 GCC 10'
+- job: compilation_CUDA11_gcc11
+  displayName: 'Compilation CUDA 11 GCC 11'
   pool:
-    vmImage: 'ubuntu-20.04'
+    vmImage: 'ubuntu-22.04'
   variables:
-    CC: gcc-10
-    CXX: g++-10
-    FC: gfortran-10
+    CC: gcc-11
+    CXX: g++-11
+    FC: gfortran-11
   steps:
   - template: .azure-pipelines/install-template.yml
     parameters:
       CUDA: true
-      CUDA_V: '11.4'
+      CUDA_V: '11.7'
   - template: .azure-pipelines/configure-template.yml
     parameters:
-      TESTFLAGS: '--with-mpi --enable-vectorization --with-cuda=cuda11'
+      TESTFLAGS: '--with-mpi --enable-vectorization --with-cuda=cuda10'
       CUDA: true
       BUILD: true
 
 - job: compilation_CUDA12
-  displayName: 'Compilation CUDA 12 GCC 10'
+  displayName: 'Compilation CUDA 12 GCC 12'
   pool:
-    vmImage: 'ubuntu-20.04'
+    vmImage: 'ubuntu-22.04'
   variables:
-    CC: gcc-10
-    CXX: g++-10
-    FC: gfortran-10
+    CC: gcc-12
+    CXX: g++-12
+    FC: gfortran-12
   steps:
   - template: .azure-pipelines/install-template.yml
     parameters:
@@ -195,7 +195,7 @@ jobs:
   - template: .azure-pipelines/install-template.yml
     parameters:
       CUDA: true
-      CUDA_V: '12.1'
+      CUDA_V: '12.9'
   - template: .azure-pipelines/configure-template.yml
     parameters:
       TESTFLAGS: '--with-mpi --enable-vectorization --with-cuda=cuda12'

.azure-pipelines/install-template.yml

@@ -6,9 +6,9 @@
 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu1804-README.md
 # ubuntu-20.04:
 # https://github.com/actions/virtual-environments/blob/main/images/linux/Ubuntu2004-README.md
-# ubuntu-22.04 "ubuntu-latest":
+# ubuntu-22.04:
 # https://github.com/actions/runner-images/blob/main/images/linux/Ubuntu2204-Readme.md
-#
+# ubuntu-24.04: "ubuntu-latest"
 parameters:
 - name: CUDA
   type: boolean
@@ -118,35 +118,38 @@ steps:
       #UBUNTU_VERSION=ubuntu2004
       ## distribution jammy: from ubuntu 22.04
       #UBUNTU_VERSION=ubuntu2204
+      ## distribution noble: from ubuntu 24.04
+      #UBUNTU_VERSION=ubuntu2204
 
       # default
-      UBUNTU_VERSION=ubuntu2004
+      UBUNTU_VERSION=ubuntu2204
 
       # CUDA_VERSION - specifies CUDA toolkit version
       echo "CUDA version: $CUDA_V"
       # http://developer.download.nvidia.com/compute/cuda/repos/
       if [ "$CUDA_V" == "6.5" ]; then
-        ## trusty
         CUDA_VERSION=6.5-14
       elif [ "$CUDA_V" == "9.2" ]; then
-        ## xenial
         CUDA_VERSION=9.2.148-1
       elif [ "$CUDA_V" == "10.2" ]; then
-        ## bionic
         CUDA_VERSION=10.2.89-1
       elif [ "$CUDA_V" == "11.4" ]; then
-        ## focal
         CUDA_VERSION=11.4.0-1
+      elif [ "$CUDA_V" == "11.7" ]; then
+        CUDA_VERSION=11.7.0-1
       elif [ "$CUDA_V" == "12.1" ]; then
-        ## focal
         CUDA_VERSION=12.1.1-1
+      elif [ "$CUDA_V" == "12.9" ]; then
+        CUDA_VERSION=12.9.0-1
       else
         # note: - on azure VM nodes with ubuntu 18.04, default gcc version is 9.3;
         #       needs at least CUDA version 10.x
         #       - on azure VM nodes with ubuntu 20.04, default gcc version is 10.3;
         #       needs at least CUDA version 11.x
-        #       - on azure VM nodes with ubuntu-latest (22.04), default gcc version is 11.3;
+        #       - on azure VM nodes with ubuntu 22.04, default gcc version is 11.3;
         #       needs at least CUDA version 11.7
+        #       - on azure VM nodes with ubuntu 24.04 (ubuntu-latest), default gcc version is 13.3;
+        #       needs at least CUDA version ?
         CUDA_VERSION=12.1.1-1
       fi
       echo

src/generate_databases/model_tomography.f90

@@ -762,7 +762,7 @@ subroutine tomo_read_next_line(unit_in,string_read)
      if (ier /= 0) stop 'error while reading tomography file'
 
      ! suppress leading white spaces, if any
-     string_read = adjustl(string_read)
+     string_read = trim(adjustl(string_read))
 
      ! suppress trailing carriage return (ASCII code 13) if any (e.g. if input text file coming from Windows/DOS)
      if (index(string_read,achar(13)) > 0) string_read = string_read(1:index(string_read,achar(13))-1)

src/generate_databases/model_vs30.f90

@@ -215,7 +215,7 @@ subroutine read_next_line(unit_in,string_read)
       if (ier /= 0) stop 'Error while reading Vs30 interface file'
 
       ! suppress leading white spaces, if any
-      string_read = adjustl(string_read)
+      string_read = trim(adjustl(string_read))
 
       ! suppress trailing carriage return (ASCII code 13) if any (e.g. if input text file coming from Windows/DOS)
       if (index(string_read,achar(13)) > 0) string_read = string_read(1:index(string_read,achar(13))-1)

src/generate_databases/save_arrays_solver.F90

@@ -143,8 +143,11 @@ subroutine save_arrays_solver_mesh()
   write(IOUT) gammazstore
   write(IOUT) jacobianstore
 
+  ! these arrays are needed for acoustic and elastic simulations (and ..)
+  ! thus we allocate it and read it in all cases (whether the simulation is acoustic, elastic, or acoustic/elastic, ..)
   write(IOUT) kappastore
   write(IOUT) mustore
+  write(IOUT) rhostore
 
   write(IOUT) ispec_is_acoustic
   write(IOUT) ispec_is_elastic
@@ -159,10 +162,6 @@ subroutine save_arrays_solver_mesh()
     write(IOUT) rmass_acoustic
   endif
 
-  ! this array is needed for acoustic simulations but also for elastic simulations with CPML,
-  ! thus we allocate it and read it in all cases (whether the simulation is acoustic, elastic, or acoustic/elastic)
-  write(IOUT) rhostore
-
   ! elastic
   if (ELASTIC_SIMULATION) then
     write(IOUT) rmass

src/generate_databases/save_arrays_solver_hdf5.F90

@@ -394,8 +394,11 @@ subroutine save_arrays_solver_mesh_hdf5()
 
     dset_name = "kappastore" ! 4 r (/0,0,0,offset_nspec/)
     call h5_create_dataset_gen(dset_name,(/NGLLX,NGLLY,NGLLZ,sum(offset_nspec(:))/), 4, CUSTOM_REAL)
-    dset_name = "mustore" ! 4 r (/0,0,0,offset_nspec/)
+    dset_name = "mustore"    ! 4 r (/0,0,0,offset_nspec/)
     call h5_create_dataset_gen(dset_name,(/NGLLX,NGLLY,NGLLZ,sum(offset_nspec(:))/), 4, CUSTOM_REAL)
+    dset_name = "rhostore"   ! 4 r (/0,0,0,offset_nspec/)
+    call h5_create_dataset_gen(dset_name,(/NGLLX,NGLLY,NGLLZ,sum(offset_nspec(:))/), 4, CUSTOM_REAL)
+
     dset_name = "ispec_is_acoustic" ! 1 l (/0,0,0,offset_nspec/)
     call h5_create_dataset_gen(dset_name,(/sum(offset_nspec(:))/), 1, 0)
     dset_name = "ispec_is_elastic" ! 1 l (/0,0,0,offset_nspec/)
@@ -409,11 +412,6 @@ subroutine save_arrays_solver_mesh_hdf5()
       call h5_create_dataset_gen(dset_name,(/sum(offset_nglob(:))/), 1, CUSTOM_REAL)
     endif
 
-    ! this array is needed for acoustic simulations but also for elastic simulations with CPML,
-    ! thus we allocate it and read it in all cases (whether the simulation is acoustic, elastic, or acoustic/elastic)
-    dset_name = "rhostore" ! 4 r (/0,0,0,offset_nspec/)
-    call h5_create_dataset_gen(dset_name,(/NGLLX,NGLLY,NGLLZ,sum(offset_nspec(:))/), 4, CUSTOM_REAL)
-
     ! elastic
     if (ELASTIC_SIMULATION) then
       dset_name = "rmass" ! 1 r (/offset_nglob/)
@@ -940,10 +938,14 @@ subroutine save_arrays_solver_mesh_hdf5()
   call h5_write_dataset_collect_hyperslab(dset_name,gammazstore,(/0,0,0,sum(offset_nspec_irregular(0:myrank-1))/),H5_COL)
   dset_name = "jacobianstore" ! 4 r (/0,0,0,offset_nspec_irregular = offset_nspec/)
   call h5_write_dataset_collect_hyperslab(dset_name,jacobianstore,(/0,0,0,sum(offset_nspec_irregular(0:myrank-1))/),H5_COL)
+
   dset_name = "kappastore" ! 4 r (/0,0,0,offset_nspec/)
   call h5_write_dataset_collect_hyperslab(dset_name,kappastore,(/0,0,0,sum(offset_nspec(0:myrank-1))/),H5_COL)
   dset_name = "mustore" ! 4 r (/0,0,0,offset_nspec/)
   call h5_write_dataset_collect_hyperslab(dset_name,mustore,(/0,0,0,sum(offset_nspec(0:myrank-1))/),H5_COL)
+  dset_name = "rhostore" ! 4 r (/0,0,0,offset_nspec/)
+  call h5_write_dataset_collect_hyperslab(dset_name, rhostore, (/0,0,0,sum(offset_nspec(0:myrank-1))/),H5_COL)
+
   dset_name = "ispec_is_acoustic" ! 1 l (/0,0,0,offset_nspec/)
   call h5_write_dataset_collect_hyperslab(dset_name, ispec_is_acoustic, (/sum(offset_nspec(0:myrank-1))/),H5_COL)
   dset_name = "ispec_is_elastic" ! 1 l (/0,0,0,offset_nspec/)
@@ -957,11 +959,6 @@ subroutine save_arrays_solver_mesh_hdf5()
     call h5_write_dataset_collect_hyperslab(dset_name, rmass_acoustic,(/sum(offset_nglob(0:myrank-1))/),H5_COL)
   endif
 
-  ! this array is needed for acoustic simulations but also for elastic simulations with CPML,
-  ! thus we allocate it and read it in all cases (whether the simulation is acoustic, elastic, or acoustic/elastic)
-  dset_name = "rhostore" ! 4 r (/0,0,0,offset_nspec/)
-  call h5_write_dataset_collect_hyperslab(dset_name, rhostore, (/0,0,0,sum(offset_nspec(0:myrank-1))/),H5_COL)
-
   ! elastic
   if (ELASTIC_SIMULATION) then
     dset_name = "rmass" ! 1 r (/offset_nglob/)

src/generate_databases/setup_color_perm.f90

@@ -631,12 +631,12 @@ subroutine setup_permutation(myrank,nspec,nglob,ibool,ANISOTROPY,perm,SAVE_MESH_
 
   call permute_elements_real(kappastore,temp_array_real,perm,nspec)
   call permute_elements_real(mustore,temp_array_real,perm,nspec)
+  call permute_elements_real(rhostore,temp_array_real,perm,nspec)
 
   if (ATTENUATION) then
      call permute_elements_real(qmu_attenuation_store,temp_array_real,perm,nspec)
      call permute_elements_real(qkappa_attenuation_store,temp_array_real,perm,nspec)
   endif
-  call permute_elements_real(rhostore,temp_array_real,perm,nspec)
 
   ! elastic arrays
   if (ELASTIC_SIMULATION) then

src/meshfem3D/meshfem3D_par.f90

@@ -90,8 +90,8 @@ module meshfem_par
 
   ! CPML
   double precision :: THICKNESS_OF_X_PML,THICKNESS_OF_Y_PML,THICKNESS_OF_Z_PML
-  logical :: ADD_PML_AS_EXTRA_MESH_LAYERS
-  integer :: NUMBER_OF_PML_LAYERS_TO_ADD
+  logical :: ADD_PML_AS_EXTRA_MESH_LAYERS = .false.  ! for mesh extension with PML layers
+  integer :: NUMBER_OF_PML_LAYERS_TO_ADD = 0
   logical, dimension(:), allocatable :: is_CPML
   integer, dimension(:), allocatable :: CPML_to_spec,CPML_regions
   integer :: nspec_CPML

src/meshfem3D/read_mesh_parameter_file.f90

@@ -197,13 +197,13 @@ subroutine read_mesh_parameter_file()
   if (THICKNESS_OF_Y_PML < 0.0) stop 'Error invalid negative value THICKNESS_OF_Y_PML'
   if (THICKNESS_OF_Z_PML < 0.0) stop 'Error invalid negative value THICKNESS_OF_Z_PML'
 
-  ! add PML as extra mesh layers
-  call read_value_logical_mesh(IIN,IGNORE_JUNK,ADD_PML_AS_EXTRA_MESH_LAYERS, 'ADD_PML_AS_EXTRA_MESH_LAYERS', ier)
-  if (ier /= 0) stop 'Error reading Mesh parameter ADD_PML_AS_EXTRA_MESH_LAYERS'
-
-  ! number of PML layers added as extra outer layers
-  call read_value_integer_mesh(IIN,IGNORE_JUNK,NUMBER_OF_PML_LAYERS_TO_ADD, 'NUMBER_OF_PML_LAYERS_TO_ADD', ier)
-  if (ier /= 0) stop 'Error reading Mesh parameter NUMBER_OF_PML_LAYERS_TO_ADD'
+  ! PML mesh extension
+  ! (optional) add PML as extra mesh layers
+  call read_value_logical_mesh_optional(IIN,ADD_PML_AS_EXTRA_MESH_LAYERS, 'ADD_PML_AS_EXTRA_MESH_LAYERS', ier)
+  if (ier /= 0) stop 'Error reading (optional) Mesh parameter ADD_PML_AS_EXTRA_MESH_LAYERS'
+  ! (optional) number of PML layers added as extra outer layers
+  call read_value_integer_mesh_optional(IIN,NUMBER_OF_PML_LAYERS_TO_ADD, 'NUMBER_OF_PML_LAYERS_TO_ADD', ier)
+  if (ier /= 0) stop 'Error reading (optional) Mesh parameter NUMBER_OF_PML_LAYERS_TO_ADD'
 
   ! user output
   if (myrank == 0) then