@@ -52,9 +52,10 @@ PROGRAM FDS
5252INTEGER :: LO10,NM,IZERO,ANG_INC_COUNTER
5353REAL (EB) :: T,DT,TNOW
5454REAL :: CPUTIME
55- REAL (EB), ALLOCATABLE , DIMENSION (:) :: TC_GLB,TC_LOC,DT_NEW,TI_LOC,TI_GLB
56- REAL (EB), ALLOCATABLE , DIMENSION (:,:) :: TC2_GLB,TC2_LOC
57- LOGICAL , ALLOCATABLE , DIMENSION (:) :: STATE_GLB,STATE_LOC
55+ REAL (EB), ALLOCATABLE , DIMENSION (:) :: TC_ARRAY,DT_NEW
56+ INTEGER , ALLOCATABLE , DIMENSION (:) :: N_VALUES
57+ REAL (EB), ALLOCATABLE , DIMENSION (:,:) :: TC2_ARRAY
58+ LOGICAL , ALLOCATABLE , DIMENSION (:) :: STATE_ARRAY
5859INTEGER :: ITER
5960TYPE (MESH_TYPE), POINTER :: M,M4
6061TYPE (OMESH_TYPE), POINTER :: M2,M3
@@ -1114,22 +1115,10 @@ SUBROUTINE MPI_INITIALIZATION_CHORES(TASK_NUMBER)
11141115
11151116 ! Set up dummy arrays to hold various arrays that must be exchanged among meshes
11161117
1117- ALLOCATE (TI_LOC(N_DEVC),STAT= IZERO)
1118- CALL ChkMemErr(' MAIN' ,' TI_LOC' ,IZERO)
1119- ALLOCATE (TI_GLB(N_DEVC),STAT= IZERO)
1120- CALL ChkMemErr(' MAIN' ,' TI_GLB' ,IZERO)
1121- ALLOCATE (STATE_GLB(2 * N_DEVC),STAT= IZERO)
1122- CALL ChkMemErr(' MAIN' ,' STATE_GLB' ,IZERO)
1123- ALLOCATE (STATE_LOC(2 * N_DEVC),STAT= IZERO)
1124- CALL ChkMemErr(' MAIN' ,' STATE_LOC' ,IZERO)
1125- ALLOCATE (TC_GLB(4 * N_DEVC),STAT= IZERO)
1126- CALL ChkMemErr(' MAIN' ,' TC_GLB' ,IZERO)
1127- ALLOCATE (TC_LOC(4 * N_DEVC),STAT= IZERO)
1128- CALL ChkMemErr(' MAIN' ,' TC_LOC' ,IZERO)
1129- ALLOCATE (TC2_GLB(2 ,N_DEVC),STAT= IZERO)
1130- CALL ChkMemErr(' MAIN' ,' TC2_GLB' ,IZERO)
1131- ALLOCATE (TC2_LOC(2 ,N_DEVC),STAT= IZERO)
1132- CALL ChkMemErr(' MAIN' ,' TC2_LOC' ,IZERO)
1118+ ALLOCATE (STATE_ARRAY(2 * N_DEVC),STAT= IZERO) ; CALL ChkMemErr(' MAIN' ,' STATE_ARRAY' ,IZERO)
1119+ ALLOCATE (TC_ARRAY(4 * N_DEVC),STAT= IZERO) ; CALL ChkMemErr(' MAIN' ,' TC_ARRAY' ,IZERO)
1120+ ALLOCATE (TC2_ARRAY(2 ,N_DEVC),STAT= IZERO) ; CALL ChkMemErr(' MAIN' ,' TC2_ARRAY' ,IZERO)
1121+ ALLOCATE (N_VALUES(N_DEVC),STAT= IZERO) ; CALL ChkMemErr(' MAIN' ,' N_VALUES' ,IZERO)
11331122
11341123
11351124 CASE (3 )
@@ -1701,7 +1690,7 @@ END SUBROUTINE STOP_CHECK
17011690
17021691SUBROUTINE END_FDS
17031692
1704- CHARACTER (255 ) :: MESSAGE
1693+ CHARACTER (MESSAGE_LENGTH ) :: MESSAGE
17051694
17061695IF (STOP_STATUS== NO_STOP .OR. STOP_STATUS== USER_STOP) CALL DUMP_TIMERS
17071696
@@ -3902,25 +3891,21 @@ SUBROUTINE EXCHANGE_GLOBAL_OUTPUTS
39023891
39033892 ! Exchange the CURRENT_STATE and PRIOR_STATE of each DEViCe
39043893
3905- STATE_LOC = .FALSE. ! _LOC is a temporary array that holds the STATE value for the devices on each node
3894+ STATE_ARRAY = .FALSE. ! Temporary array that holds the STATE value for the devices on each node
39063895 DO NM= LOWER_MESH_INDEX,UPPER_MESH_INDEX
39073896 DO N= 1 ,N_DEVC
39083897 DV = > DEVICE(N)
39093898 IF (DV% MESH== NM) THEN
3910- STATE_LOC (N) = DV% CURRENT_STATE
3911- STATE_LOC (N+ N_DEVC) = DV% PRIOR_STATE
3899+ STATE_ARRAY (N) = DV% CURRENT_STATE
3900+ STATE_ARRAY (N+ N_DEVC) = DV% PRIOR_STATE
39123901 ENDIF
39133902 ENDDO
39143903 ENDDO
3915- IF (N_MPI_PROCESSES> 1 ) THEN
3916- CALL MPI_ALLREDUCE(STATE_LOC(1 ),STATE_GLB(1 ),2 * N_DEVC,MPI_LOGICAL,MPI_LXOR,MPI_COMM_WORLD,IERR)
3917- ELSE
3918- STATE_GLB = STATE_LOC
3919- ENDIF
3904+ IF (N_MPI_PROCESSES> 1 ) CALL MPI_ALLREDUCE(MPI_IN_PLACE,STATE_ARRAY,2 * N_DEVC,MPI_LOGICAL,MPI_LXOR,MPI_COMM_WORLD,IERR)
39203905 DO N= 1 ,N_DEVC
39213906 DV = > DEVICE(N)
3922- DV% CURRENT_STATE = STATE_GLB (N)
3923- DV% PRIOR_STATE = STATE_GLB (N+ N_DEVC)
3907+ DV% CURRENT_STATE = STATE_ARRAY (N)
3908+ DV% PRIOR_STATE = STATE_ARRAY (N+ N_DEVC)
39243909 ENDDO
39253910
39263911 ! Dry pipe sprinkler logic
@@ -3943,70 +3928,84 @@ SUBROUTINE EXCHANGE_GLOBAL_OUTPUTS
39433928 ! For OP_INDEX=2 and 3, we take the MIN or MAX of all the VALUEs, along with the MINLOC or MAXLOC.
39443929
39453930 OPERATION_LOOP: DO OP_INDEX= 1 ,3
3931+
39463932 IF (OP_INDEX== 2 .AND. .NOT. MIN_DEVICES_EXIST) CYCLE OPERATION_LOOP
39473933 IF (OP_INDEX== 3 .AND. .NOT. MAX_DEVICES_EXIST) CYCLE OPERATION_LOOP
3934+ N_VALUES = 0
39483935 SELECT CASE (OP_INDEX)
3949- CASE (1 ) ; TC_LOC = 0._EB ; MPI_OP_INDEX = MPI_SUM ; DIM_FAC = 4
3950- CASE (2 ) ; TC2_LOC = 1.E10_EB ; MPI_OP_INDEX = MPI_MINLOC ; DIM_FAC = 1
3951- CASE (3 ) ; TC2_LOC = - 1.E10_EB ; MPI_OP_INDEX = MPI_MAXLOC ; DIM_FAC = 1
3936+ CASE (1 ) ; TC_ARRAY = 0._EB ; MPI_OP_INDEX = MPI_SUM ; DIM_FAC = 4
3937+ CASE (2 ) ; TC2_ARRAY = 1.E10_EB ; MPI_OP_INDEX = MPI_MINLOC ; DIM_FAC = 1
3938+ CASE (3 ) ; TC2_ARRAY = - 1.E10_EB ; MPI_OP_INDEX = MPI_MAXLOC ; DIM_FAC = 1
39523939 END SELECT
3940+
39533941 DEVICE_LOOP_1: DO N= 1 ,N_DEVC
39543942 DV = > DEVICE(N)
3955- IF (OP_INDEX== 1 .AND. (DV% SPATIAL_STATISTIC(1 :3 )==' MIN' .OR. DV% SPATIAL_STATISTIC(1 :3 )==' MAX' )) CYCLE
3956- IF (OP_INDEX== 2 .AND. DV% SPATIAL_STATISTIC(1 :3 )/= ' MIN' ) CYCLE
3957- IF (OP_INDEX== 3 .AND. DV% SPATIAL_STATISTIC(1 :3 )/= ' MAX' ) CYCLE
3943+ IF (OP_INDEX== 1 .AND. (DV% SPATIAL_STATISTIC(1 :3 )==' MIN' .OR. DV% SPATIAL_STATISTIC(1 :3 )==' MAX' )) CYCLE DEVICE_LOOP_1
3944+ IF (OP_INDEX== 2 .AND. DV% SPATIAL_STATISTIC(1 :3 )/= ' MIN' ) CYCLE DEVICE_LOOP_1
3945+ IF (OP_INDEX== 3 .AND. DV% SPATIAL_STATISTIC(1 :3 )/= ' MAX' ) CYCLE DEVICE_LOOP_1
39583946 DO NN= 1 ,DV% N_SUBDEVICES
39593947 SDV = > DV% SUBDEVICE(NN)
3948+ N_VALUES(N) = N_VALUES(N) + SDV% N_VALUES
39603949 SELECT CASE (OP_INDEX)
39613950 CASE (1 )
3962- TC_LOC (N) = TC_LOC (N) + SDV% VALUE_1
3963- TC_LOC (N+ N_DEVC) = TC_LOC (N+ N_DEVC) + SDV% VALUE_2
3964- TC_LOC (N+2 * N_DEVC) = TC_LOC (N+2 * N_DEVC) + SDV% VALUE_3
3965- TC_LOC (N+3 * N_DEVC) = TC_LOC (N+3 * N_DEVC) + SDV% VALUE_4
3951+ TC_ARRAY (N) = TC_ARRAY (N) + SDV% VALUE_1
3952+ TC_ARRAY (N+ N_DEVC) = TC_ARRAY (N+ N_DEVC) + SDV% VALUE_2
3953+ TC_ARRAY (N+2 * N_DEVC) = TC_ARRAY (N+2 * N_DEVC) + SDV% VALUE_3
3954+ TC_ARRAY (N+3 * N_DEVC) = TC_ARRAY (N+3 * N_DEVC) + SDV% VALUE_4
39663955 CASE (2 )
3967- IF (SDV% VALUE_1< TC2_LOC (1 ,N)) THEN
3968- TC2_LOC (1 ,N) = SDV% VALUE_1
3969- TC2_LOC (2 ,N) = SDV% VALUE_2
3956+ IF (SDV% VALUE_1< TC2_ARRAY (1 ,N)) THEN
3957+ TC2_ARRAY (1 ,N) = SDV% VALUE_1
3958+ TC2_ARRAY (2 ,N) = SDV% VALUE_2
39703959 ENDIF
39713960 CASE (3 )
3972- IF (SDV% VALUE_1> TC2_LOC (1 ,N)) THEN
3973- TC2_LOC (1 ,N) = SDV% VALUE_1
3974- TC2_LOC (2 ,N) = SDV% VALUE_2
3961+ IF (SDV% VALUE_1> TC2_ARRAY (1 ,N)) THEN
3962+ TC2_ARRAY (1 ,N) = SDV% VALUE_1
3963+ TC2_ARRAY (2 ,N) = SDV% VALUE_2
39753964 ENDIF
39763965 END SELECT
39773966 ENDDO
39783967 ENDDO DEVICE_LOOP_1
3968+
3969+ ! Perform MPI exchanges to sum or take max/min of device values collected on meshes controlled by different processes
3970+
39793971 IF (N_MPI_PROCESSES> 1 ) THEN
3972+ CALL MPI_ALLREDUCE(MPI_IN_PLACE,N_VALUES,N_DEVC,MPI_INTEGER,MPI_SUM,MPI_COMM_WORLD,IERR)
39803973 SELECT CASE (OP_INDEX)
3981- CASE (1 ) ; CALL MPI_ALLREDUCE(TC_LOC(1 ),TC_GLB(1 ),DIM_FAC* N_DEVC,MPI_DOUBLE_PRECISION,MPI_OP_INDEX,MPI_COMM_WORLD,IERR)
3982- CASE (2 :3 ) ; CALL MPI_ALLREDUCE(TC2_LOC(1 ,1 ),TC2_GLB(1 ,1 ),N_DEVC,MPI_2DOUBLE_PRECISION,MPI_OP_INDEX,MPI_COMM_WORLD,IERR)
3983- END SELECT
3984- ELSE
3985- SELECT CASE (OP_INDEX)
3986- CASE (1 ) ; TC_GLB = TC_LOC
3987- CASE (2 :3 ) ; TC2_GLB = TC2_LOC
3974+ CASE (1 )
3975+ CALL MPI_ALLREDUCE(MPI_IN_PLACE,TC_ARRAY,DIM_FAC* N_DEVC,MPI_DOUBLE_PRECISION,MPI_OP_INDEX,MPI_COMM_WORLD,IERR)
3976+ CASE (2 :3 )
3977+ CALL MPI_ALLREDUCE(MPI_IN_PLACE,TC2_ARRAY,N_DEVC,MPI_2DOUBLE_PRECISION,MPI_OP_INDEX,MPI_COMM_WORLD,IERR)
39883978 END SELECT
39893979 ENDIF
3980+
3981+ ! Put summed values from the subdevices (SDV) back into the controlling device (DV)
3982+
39903983 DEVICE_LOOP_2: DO N= 1 ,N_DEVC
3984+
39913985 DV = > DEVICE(N)
3992- IF (OP_INDEX== 1 .AND. (DV% SPATIAL_STATISTIC(1 :3 )==' MIN' .OR. DV% SPATIAL_STATISTIC(1 :3 )==' MAX' )) CYCLE
3993- IF (OP_INDEX== 2 .AND. DV% SPATIAL_STATISTIC(1 :3 )/= ' MIN' ) CYCLE
3994- IF (OP_INDEX== 3 .AND. DV% SPATIAL_STATISTIC(1 :3 )/= ' MAX' ) CYCLE
3986+ IF (OP_INDEX== 1 .AND. (DV% SPATIAL_STATISTIC(1 :3 )==' MIN' .OR. DV% SPATIAL_STATISTIC(1 :3 )==' MAX' )) CYCLE DEVICE_LOOP_2
3987+ IF (OP_INDEX== 2 .AND. DV% SPATIAL_STATISTIC(1 :3 )/= ' MIN' ) CYCLE DEVICE_LOOP_2
3988+ IF (OP_INDEX== 3 .AND. DV% SPATIAL_STATISTIC(1 :3 )/= ' MAX' ) CYCLE DEVICE_LOOP_2
3989+ IF (MY_RANK== 0 .AND. N_VALUES(N)==0 .AND. DV% SPATIAL_STATISTIC/= ' null' ) &
3990+ WRITE (LU_ERR,' (3A)' ) ' WARNING: DEVC ' ,TRIM (DV% ID),' has no values.'
39953991 IF (OP_INDEX== 1 ) THEN
3996- DV% VALUE_1 = TC_GLB (N)
3997- DV% VALUE_2 = TC_GLB ( N_DEVC+ N)
3998- DV% VALUE_3 = TC_GLB (2 * N_DEVC+ N)
3999- DV% VALUE_4 = TC_GLB (3 * N_DEVC+ N)
3992+ DV% VALUE_1 = TC_ARRAY (N)
3993+ DV% VALUE_2 = TC_ARRAY ( N_DEVC+ N)
3994+ DV% VALUE_3 = TC_ARRAY (2 * N_DEVC+ N)
3995+ DV% VALUE_4 = TC_ARRAY (3 * N_DEVC+ N)
40003996 ENDIF
40013997 IF (OP_INDEX> 1 .AND. (DV% SPATIAL_STATISTIC==' MIN' .OR. DV% SPATIAL_STATISTIC==' MAX' )) THEN
4002- DV% VALUE_1 = TC2_GLB (1 ,N)
3998+ DV% VALUE_1 = TC2_ARRAY (1 ,N)
40033999 ENDIF
4000+
4001+ ! Special case for MINLOC or MAXLOC
4002+
40044003 IF (OP_INDEX> 1 .AND. (DV% SPATIAL_STATISTIC(1 :6 )==' MINLOC' .OR. DV% SPATIAL_STATISTIC(1 :6 )==' MAXLOC' )) THEN
40054004 NO_NEED_TO_RECV = .FALSE.
40064005 DO NN= 1 ,DV% N_SUBDEVICES
40074006 SDV = > DV% SUBDEVICE(NN)
40084007 IF (PROCESS(SDV% MESH)==MY_RANK) THEN
4009- IF (SDV% MESH== NINT (TC2_GLB (2 ,N))) THEN
4008+ IF (SDV% MESH== NINT (TC2_ARRAY (2 ,N))) THEN
40104009 DV% VALUE_1 = SDV% VALUE_3
40114010 IF (MY_RANK> 0 ) THEN
40124011 CALL MPI_SEND(DV% VALUE_1,1 ,MPI_DOUBLE_PRECISION,0 ,999 ,MPI_COMM_WORLD,IERR)
@@ -4021,7 +4020,9 @@ SUBROUTINE EXCHANGE_GLOBAL_OUTPUTS
40214020 CALL MPI_BARRIER(MPI_COMM_WORLD,IERR)
40224021 CALL MPI_BCAST(DV% VALUE_1,1 ,MPI_DOUBLE_PRECISION,0 ,MPI_COMM_WORLD,IERR)
40234022 ENDIF
4023+
40244024 ENDDO DEVICE_LOOP_2
4025+
40254026 ENDDO OPERATION_LOOP
40264027
40274028ENDIF EXCHANGE_DEVICE
0 commit comments