Merge pull request #2740 from OpenFAST/rc-4.0.3

Release 4.0.3

Author: andrew-platt

cmake/OpenfastFortranOptions.cmake

@@ -158,13 +158,18 @@ endmacro(set_fast_intel_fortran)
 # arch
 #
 macro(set_fast_intel_fortran_posix)
-  # Set size where temporary are stored on heap instead of stack
-  #   1000: size in kB (1 MB)
-  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fpic -fpp -heap-arrays 1000")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} -fpic -fpp")
 
   # debug flags
   if(CMAKE_BUILD_TYPE MATCHES Debug)
-    set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -check all,noarg_temp_created -traceback -init=huge,infinity" )
+    if(${CMAKE_Fortran_COMPILER_ID} MATCHES "IntelLLVM")
+      # NOTE: there is a bug in the 2024 and 2025 IFX compiler causing conflicts between the `check:uninit` and `-lm -ldl` flags
+      #     When this is fixed in IFX, we will want to update this to check against versions before fix
+      #     See here: https://community.intel.com/t5/Intel-Fortran-Compiler/ifx-IFX-2023-2-0-20230721-linker-problems-with-check-uninit/m-p/1527816
+      set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -check all,noarg_temp_created,nouninit -traceback -init=huge,infinity" )
+    else()
+      set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} -check all,noarg_temp_created -traceback -init=huge,infinity" )
+    endif()
   endif()
 
   # If double precision, make real and double constants 64 bits
@@ -203,9 +208,7 @@ macro(set_fast_intel_fortran_windows)
   # Turn off specific warnings
   # - 5199: too many continuation lines
   # - 5268: 132 column limit
-  # Set size where temporary are stored on heap instead of stack
-  #   1000: size in kB (1 MB)
-  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} /Qdiag-disable:5199,5268 /fpp /heap-arrays:1000")
+  set(CMAKE_Fortran_FLAGS "${CMAKE_Fortran_FLAGS} /Qdiag-disable:5199,5268 /fpp")
 
   # If double precision, make constants double precision
   if (DOUBLE_PRECISION)
@@ -222,7 +225,11 @@ macro(set_fast_intel_fortran_windows)
 
   # debug flags
   if(CMAKE_BUILD_TYPE MATCHES Debug)
-    set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} /check:all,noarg_temp_created /traceback /Qinit=huge,infinity" )
+    if(${CMAKE_Fortran_COMPILER_ID} MATCHES "IntelLLVM")
+      set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} /check:all,noarg_temp_created,nouninit /traceback /Qinit=huge,infinity" )
+    else()
+      set( CMAKE_Fortran_FLAGS_DEBUG "${CMAKE_Fortran_FLAGS_DEBUG} /check:all,noarg_temp_created /traceback /Qinit=huge,infinity" )
+    endif()
   endif()
 
   check_f2008_features()

docs/changelogs/ReleaseProcess.md

@@ -49,33 +49,33 @@ After posting and tagging release
    * Run one of the executables and check the version info. Muck about with VS if there is an issue.
    * Also run `dumpbin.exe /dependents <exe>.exe` to check static linking
    * NOTE: build the simulink last -- it messes up some things otherwise
-    - [ ] AeroDisk_Driver_x64.exe
-    - [ ] AeroDyn_Driver_x64.exe
-    - [ ] AeroDyn_Driver_x64_OpenMP.exe
-    - [ ] AeroDyn_Inflow_C_Binding_x64.dll
-    - [ ] AeroDyn_Inflow_C_Binding_x64_OpenMP.dll
-    - [ ] BeamDyn_Driver_x64.exe
-    - [ ] DISCON.dll (x64)
-    - [ ] DISCON_ITIBarge.dll (x64)
-    - [ ] DISCON_OC3Hywind.dll (x64)
-    - [ ] DISCON_SC.dll (x64)
-    - [ ] FAST.Farm_x64.exe
-    - [ ] FAST.Farm_x64_OMP.exe
-    - [ ] FAST_SFunc.mexw64 -- build from MATLAB
-    - [ ] HydroDynDriver_x64.exe
-    - [ ] HydroDyn_C_Binding_x64.dll
-    - [ ] IfW_C_Binding_x64.dll
-    - [ ] InflowWind_Driver_x64.exe
-    - [ ] InflowWind_Driver_x64_OpenMP.exe
-    - [ ] MoorDyn_Driver_x64.exe
-    - [ ] MoorDyn_C_Binding_x64.dll
-    - [ ] OpenFAST-Simulink_x64.dll -- change `additional dependencies` in the `OpenFAST-Simulink` project in `FAST` to point to correct install of MATLAB
-    - [ ] openfast_x64.exe
-    - [ ] SeaStateDriver_x64.exe
-    - [ ] SimpleElastoDyn_x64.exe
-    - [ ] SubDyn_x64.exe
-    - [ ] Turbsim_x64.exe
-    - [ ] UnsteadyAero_x64.exe
+    - [ ] `AeroDisk_Driver_x64.exe`
+    - [ ] `AeroDyn_Driver_x64.exe`
+    - [ ] `AeroDyn_Driver_x64_OpenMP.exe`
+    - [ ] `AeroDyn_Inflow_c_binding_x64.dll`
+    - [ ] `AeroDyn_Inflow_c_binding_x64_OpenMP.dll`
+    - [ ] `BeamDyn_Driver_x64.exe`
+    - [ ] `DISCON.dll` (x64)
+    - [ ] `DISCON_ITIBarge.dll` (x64)
+    - [ ] `DISCON_OC3Hywind.dll` (x64)
+    - [ ] `DISCON_SC.dll` (x64)
+    - [ ] `FAST.Farm_x64.exe`
+    - [ ] `FAST.Farm_x64_OMP.exe`
+    - [ ] `FAST_SFunc.mexw64` -- build from MATLAB
+    - [ ] `HydroDynDriver_x64.exe`
+    - [ ] `HydroDyn_c_binding_x64.dll`
+    - [ ] `InflowWind_c_binding_x64.dll`
+    - [ ] `InflowWind_Driver_x64.exe`
+    - [ ] `InflowWind_Driver_x64_OpenMP.exe`
+    - [ ] `MoorDyn_Driver_x64.exe`
+    - [ ] `MoorDyn_c_binding_x64.dll`
+    - [ ] `OpenFAST-Simulink_x64.dll` -- change `additional dependencies` in the `OpenFAST-Simulink` project in `FAST` to point to correct install of MATLAB
+    - [ ] `openfast_x64.exe`
+    - [ ] `SeaStateDriver_x64.exe`
+    - [ ] `SimpleElastoDyn_x64.exe`
+    - [ ] `SubDyn_x64.exe`
+    - [ ] `Turbsim_x64.exe`
+    - [ ] `UnsteadyAero_x64.exe`
 5. Upload all filesUnset the no tracking of files
    ```
    git ls-files -v | grep "^[a-z]"

docs/changelogs/v4.0.3.md

@@ -0,0 +1,91 @@
+**Feature or improvement description**
+Pull request to merge `rc-4.0.3` into `main` and create a tagged release for v4.0.3
+
+See the milestone and project pages for additional information
+
+    https://github.com/OpenFAST/openfast/milestone/19
+
+Test results, if applicable
+See GitHub Actions
+
+### Release checklist:
+- [ ] Update the documentation version in docs/conf.py
+- [ ] Update the versions in docs/source/user/api\_change.rst
+- [ ] Update version info in openfast\_io/pyproject.toml
+- [ ] Verify readthedocs builds correctly
+- [ ] Create an annotated tag in OpenFAST during merge (mark as most recent if necessary)
+- [ ] Create a merge commit in r-test and add a corresponding annotated tag
+- [ ] Compile executables for Windows builds
+    - [ ] `AeroDisk_Driver_x64.exe`
+    - [ ] `AeroDyn_Driver_x64.exe`
+    - [ ] `AeroDyn_Driver_x64_OpenMP.exe`
+    - [ ] `AeroDyn_Inflow_c_binding_x64.dll`
+    - [ ] `AeroDyn_Inflow_c_binding_x64_OpenMP.dll`
+    - [ ] `BeamDyn_Driver_x64.exe`
+    - [ ] `DISCON.dll (x64)`
+    - [ ] `DISCON_ITIBarge.dll (x64)`
+    - [ ] `DISCON_OC3Hywind.dll (x64)`
+    - [ ] `DISCON_SC.dll (x64)`
+    - [ ] `FAST.Farm_x64.exe`
+    - [ ] `FAST.Farm_x64_OMP.exe`
+    - [ ] `FAST_SFunc.mexw64`
+    - [ ] `HydroDynDriver_x64.exe`
+    - [ ] `HydroDyn_C_Binding_x64.dll`
+    - [ ] `IinflowWind_c_binding_x64.dll`
+    - [ ] `InflowWind_Driver_x64.exe`
+    - [ ] `InflowWind_Driver_x64_OpenMP.exe`
+    - [ ] `MoorDyn_Driver_x64.exe`
+    - [ ] `MoorDyn_c_binding_x64.dll`
+    - [ ] `OpenFAST-Simulink_x64.dll`
+    - [ ] `openfast_x64.exe`
+    - [ ] `SeaStateDriver_x64.exe`
+    - [ ] `SimpleElastoDyn_x64.exe`
+    - [ ] `SubDyn_x64.exe`
+    - [ ] `Turbsim_x64.exe`
+    - [ ] `UnsteadyAero_x64.exe`
+
+# Changelog
+
+## Overview
+
+This release includes several bug fixes and improvements for _FAST.Farm_, including segmentation faults with large farms, file I/O collision fixes, increased error message length, and build/compile issues.
+
+## General
+
+### CMake build system
+
+#2709 Add "nouninit" to debug flags for IntelLLVM (@andrew-platt)
+
+#2732 Disable use of heap arrays for the Intel fortran compilers (@deslaughter)
+
+
+### openfast_io
+
+#2727 generate BD files if fst_vt dictionary BD keys are not empty (@mayankchetan)
+
+
+### Visual Studio build
+
+#2742 Fix VS build for FAST when using IFX Complier (@deslaughter)
+
+
+## Solvers
+
+### FAST.Farm
+
+#2711 Fix FAST.Farm issues with OMP (segfaults mostly) (@andrew-platt)
+
+
+## Module changes
+
+### NWTC library
+
+#2710 Increase length of error messages (@andrew-platt)
+
+#2741 bug in simulation status number of days left (@andrew-platt)
+
+
+## Input file changes
+
+No input file changes are required from 4.0.2.
+

docs/conf.py

@@ -138,7 +138,7 @@ def runDoxygen(sourcfile, doxyfileIn, doxyfileOut):
 # The short X.Y version.
 version = u'4.0'
 # The full version, including alpha/beta/rc tags.
-release = u'v4.0.2'
+release = u'v4.0.3'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.

docs/source/user/api_change.rst

@@ -10,6 +10,12 @@ The line number corresponds to the resulting line number after all changes are i
 Thus, be sure to implement each in order so that subsequent line numbers are correct.
 
 
+OpenFAST v4.0.2 to OpenFAST v4.0.3
+----------------------------------
+
+No input file changes were made.
+
+
 OpenFAST v4.0.1 to OpenFAST v4.0.2
 ----------------------------------
 

docs/source/user/fast.farm/Pictures/Troubleshooting_SegFaultScreen.png

@@ -14,3 +14,36 @@ available in the :ref:`installation` documentation.
 
 .. note::
    Checkpoint-restart capability has not yet been implemented within FAST.Farm.
+
+
+Troubleshooting
+---------------
+
+Segmentation fault at start
+~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Very large FAST.Farm simulations may experience a segmentation fault if
+insufficient stack memory is set aside for OpenMP.  This may appear on
+simulations with more than 50 turbines spaced over tens of kilometers.  If this
+is the case, the fault is likely to occur right after screen displays the T=0
+timestep.
+
+.. image:: Pictures/Troubleshooting_SegFaultScreen.png
+
+To increase the stack assigned to the OpenMP process on linux based machines,
+the environment variable `OMP_STACKSIZE` can be increased from the default 4 MB
+(Intel compiled) to 32 MB (or more) by
+
+.. code-block::
+
+      export OMP_STACKSIZE="32 M"
+
+If this solves the segmentation fault, then the root cause is from the
+parallelization with OpenMP not having sufficient stack reserve.  If this does
+not solve the issue, then there may be other problems that should be reported.
+
+For further reading on segmentation faults with OpenMP, see `stackoverflow
+comment
+<https://stackoverflow.com/questions/13264274/why-segmentation-fault-is-happening-in-this-openmp-code/13266595#13266595>`_
+and `Intel OpenMP documentation
+<https://www.intel.com/content/www/us/en/developer/articles/troubleshooting/openmp-stacksize-common-error.html>`_.

docs/source/user/fast.farm/RunningFFarm.rst

@@ -701,6 +701,13 @@ subroutine HighResGridCalcOutput(n, u, p, xd, y, m, errStat, errMsg)
    ! NOTE: loop here is different from low res grid, doing: turbines > grid > turbines(nt/=nt2) > planes
    ! instead of grid > turbines > planes
    ! TODO explain 
+   !
+   ! WARNING: the way this is setup can use a lot of memory, and may run out of OMP stack.  If that happens,
+   !           use `export OMP_STACKSIZE="32 M"` (default is 4 M).
+   !     Rough calculation of memory expected:
+   !           maxN_wake * 13 * OMP_NUM_THREADS * <precision> = size in bytes
+   !     HOWEVER, real world testing shows that for 103 threads with 114 turbines and maxN_wake=101346 is more like
+   !           maxN_wake * 40 * <precision> = size in bytes
    NumGrid_high  = p%nX_high*p%nY_high*p%nZ_high
 
    do nt = 1,p%NumTurbines
@@ -779,7 +786,7 @@ end subroutine HighResGridCalcOutput
 subroutine AWAE_Init( InitInp, u, p, x, xd, z, OtherState, y, m, Interval, InitOut, errStat, errMsg )
    type(AWAE_InitInputType),       intent(in   ) :: InitInp       !< Input data for initialization routine
    type(AWAE_InputType),           intent(  out) :: u             !< An initial guess for the input; input mesh must be defined
-   type(AWAE_ParameterType),target,intent(  out) :: p             !< Parameters
+   type(AWAE_ParameterType),       intent(  out) :: p             !< Parameters
    type(AWAE_ContinuousStateType), intent(  out) :: x             !< Initial continuous states
    type(AWAE_DiscreteStateType),   intent(  out) :: xd            !< Initial discrete states
    type(AWAE_ConstraintStateType), intent(  out) :: z             !< Initial guess of the constraint states
@@ -799,7 +806,7 @@ subroutine AWAE_Init( InitInp, u, p, x, xd, z, OtherState, y, m, Interval, InitO
    character(ErrMsgLen)                          :: errMsg2       ! temporary error message
    character(*), parameter                       :: RoutineName = 'AWAE_Init'
    type(InflowWind_InitInputType)                :: IfW_InitInp
-   type(InflowWind_InitOutputType), target       :: IfW_InitOut
+   type(InflowWind_InitOutputType)               :: IfW_InitOut
 
       ! Initialize variables for this routine
    errStat = ErrID_None
@@ -1112,12 +1119,9 @@ subroutine CheckModAmb3Boundaries()
       real(ReKi)                       :: ff_lim(2)
       real(ReKi)                       :: hr_lim(2)
       real(ReKi),          parameter   :: GridTol = 1.0E-3  ! Tolerance from IfW for checking the high-res grid (Mod_AmbWind=3 only).
-      type(FlowFieldType), pointer     :: ff                ! alias to shorten notation to fullfield
-      type(WindFileDat),   pointer     :: wfi               ! alias to shorten notation to WindFileInfo
       character(1024)                  :: tmpMsg
 
-      ff  => p%IfW(nt)%FlowField 
-      wfi => IfW_InitOut%WindFileInfo
+      associate(ff => p%IfW(nt)%FlowField, wfi => IfW_InitOut%WindFileInfo)
 
       tmpMsg = NewLine//NewLine//'Turbine '//trim(Num2LStr(nt))//' -- Mod_AmbWind=3 requires the FAST.Farm high-res grid '// &
                'is entirely contained within the flow-field from InflowWind. '//NewLine//' Try setting:'//NewLine
@@ -1192,6 +1196,7 @@ subroutine CheckModAmb3Boundaries()
             ErrMsg2  = NewLine//NewLine//'Turbine '//trim(Num2LStr(nt))//' -- Mod_AmbWind=3 requires InflowWind propagation direction alignment with X or Y (0, 90, 180, 270 degrees).'
          endif
       endif
+      end associate
    end subroutine CheckModAmb3Boundaries
 
    logical function Failed()
@@ -1288,9 +1293,6 @@ subroutine AWAE_UpdateStates( t, n, u, p, x, xd, z, OtherState, m, errStat, errM
    errMsg  = ""
 
    ! Read the ambient wind data that is needed for t+dt, i.e., n+1
-!#ifdef _OPENMP
-!   t1 = omp_get_wtime()  
-!#endif 
 
    if ( (n+1) == (p%NumDT-1) ) then
       n_high_low = 0
@@ -1301,10 +1303,6 @@ subroutine AWAE_UpdateStates( t, n, u, p, x, xd, z, OtherState, m, errStat, errM
    if ( p%Mod_AmbWind == 1 ) then
          ! read from file the ambient flow for the n+1 time step
       call ReadLowResWindFile(n+1, p, m%Vamb_Low, errStat2, errMsg2);   if (Failed()) return;
-   !#ifdef _OPENMP
-   !   t2 = omp_get_wtime()      
-   !   write(*,*) '        AWAE_UpdateStates: Time spent reading Low Res data : '//trim(num2lstr(t2-t1))//' seconds'            
-   !#endif   
 
       !$OMP PARALLEL DO DEFAULT(Shared) PRIVATE(nt, i_hl, errStat2, errMsg2) !Private(nt,tm2,tm3)
       do nt = 1,p%NumTurbines
@@ -1400,11 +1398,6 @@ subroutine AWAE_UpdateStates( t, n, u, p, x, xd, z, OtherState, m, errStat, errM
       xd%WAT_B_Box(1:3) = xd%WAT_B_Box(1:3) + xd%Ufarm(1:3)*real(p%dt_low,ReKi)
    endif
 
-!#ifdef _OPENMP
-!   t1 = omp_get_wtime()      
-!   write(*,*) '        AWAE_UpdateStates: Time spent reading High Res data : '//trim(num2lstr(t1-t2))//' seconds'             
-!#endif
-
 contains
    logical function Failed()
       call SetErrStat( ErrStat2, ErrMsg2, ErrStat, ErrMsg, RoutineName)

modules/awae/src/AWAE.f90

@@ -513,7 +513,9 @@ SUBROUTINE AWAE_PrintSum(  p, u, y, ErrStat, ErrMsg )
 
 
 
+   !$OMP critical(fileopen_critical)
    CLOSE(UnSu)
+   !$OMP end critical(fileopen_critical)
 
 RETURN
 END SUBROUTINE AWAE_PrintSum

modules/awae/src/AWAE_IO.f90

@@ -33,7 +33,7 @@ MODULE NWTC_Base
       ! General constants:
 
    INTEGER, PARAMETER            :: BITS_IN_ADDR  = C_INTPTR_T*8                  !< The number of bits in an address (32-bit or 64-bit).
-   INTEGER, PARAMETER            :: ErrMsgLen = 1024                              !< The maximum number of characters in an error message in the FAST framework
+   INTEGER, PARAMETER            :: ErrMsgLen = 8196                              !< The maximum number of characters in an error message in the FAST framework
 
    INTEGER(IntKi), PARAMETER     :: ChanLen   = 20                                !< The maximum allowable length of channel names (i.e., width of output columns) in the FAST framework
    INTEGER(IntKi), PARAMETER     :: OutStrLenM1 = ChanLen - 1                     !< The maximum allowable length of channel names without optional "-" or "M" at the beginning to indicate the negative of the channel