From b4ef17f1001c3b34270e4671478fad5c894a536a Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Tue, 29 Oct 2024 10:45:02 -0700 Subject: [PATCH 1/7] CI: find and print backtraces --- .azure-pipelines.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index bdcfe1c9864..7bea93fbc7e 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -167,4 +167,10 @@ jobs: # run tests (exclude pytest.AMReX when running Python tests) ctest --test-dir build --output-on-failure -E AMReX + + # find and print backtraces + find build/bin/ -type f -name "Backtrace*" \ + -exec echo -e "\nBacktrace\n---------\n{}\n---------" \; \ + -exec cat {} \; \ + -exec echo -e "\n" \; displayName: 'Test' From 29c72c9e088f7dec7ea7dbde93e023e8983b5c9f Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 29 Oct 2024 11:42:26 -0700 Subject: [PATCH 2/7] Inject NaN to trigger crash and backtrace --- .../PsatdAlgorithmComoving.cpp | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp index f809c3f2c20..ae6e007053a 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp @@ -138,24 +138,24 @@ PsatdAlgorithmComoving::pushSpectralFields (SpectralFieldData& f) const const Complex X4 = X4_arr(i,j,k); // Update E - fields(i,j,k,Idx.Ex) = C*Ex_old + S_ck*c2*I*(ky_mod*Bz_old - kz_mod*By_old) - + X4*Jx - I*(X2*rho_new - X3*rho_old)*kx_mod; + fields(i,j,k,Idx.Ex) = std::nan(""); //C*Ex_old + S_ck*c2*I*(ky_mod*Bz_old - kz_mod*By_old) + //+ X4*Jx - I*(X2*rho_new - X3*rho_old)*kx_mod; - fields(i,j,k,Idx.Ey) = C*Ey_old + S_ck*c2*I*(kz_mod*Bx_old - kx_mod*Bz_old) - + X4*Jy - I*(X2*rho_new - X3*rho_old)*ky_mod; + fields(i,j,k,Idx.Ey) = std::nan(""); //C*Ey_old + S_ck*c2*I*(kz_mod*Bx_old - kx_mod*Bz_old) + //+ X4*Jy - I*(X2*rho_new - X3*rho_old)*ky_mod; - fields(i,j,k,Idx.Ez) = C*Ez_old + S_ck*c2*I*(kx_mod*By_old - ky_mod*Bx_old) - + X4*Jz - I*(X2*rho_new - X3*rho_old)*kz_mod; + fields(i,j,k,Idx.Ez) = std::nan(""); //C*Ez_old + S_ck*c2*I*(kx_mod*By_old - ky_mod*Bx_old) + //+ X4*Jz - I*(X2*rho_new - X3*rho_old)*kz_mod; // Update B - fields(i,j,k,Idx.Bx) = C*Bx_old - S_ck*I*(ky_mod*Ez_old - kz_mod*Ey_old) - + X1*I*(ky_mod*Jz - kz_mod*Jy); + fields(i,j,k,Idx.Bx) = std::nan(""); //C*Bx_old - S_ck*I*(ky_mod*Ez_old - kz_mod*Ey_old) + //+ X1*I*(ky_mod*Jz - kz_mod*Jy); - fields(i,j,k,Idx.By) = C*By_old - S_ck*I*(kz_mod*Ex_old - kx_mod*Ez_old) - + X1*I*(kz_mod*Jx - kx_mod*Jz); + fields(i,j,k,Idx.By) = std::nan(""); //C*By_old - S_ck*I*(kz_mod*Ex_old - kx_mod*Ez_old) + //+ X1*I*(kz_mod*Jx - kx_mod*Jz); - fields(i,j,k,Idx.Bz) = C*Bz_old - S_ck*I*(kx_mod*Ey_old - ky_mod*Ex_old) - + X1*I*(kx_mod*Jy - ky_mod*Jx); + fields(i,j,k,Idx.Bz) = std::nan(""); //C*Bz_old - S_ck*I*(kx_mod*Ey_old - ky_mod*Ex_old) + //+ X1*I*(kx_mod*Jy - ky_mod*Jx); }); } } From 4abc5670eb32b039ca63d098ce1d2410c02aee46 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:40:41 -0700 Subject: [PATCH 3/7] Handle CTest exit status correctly --- .azure-pipelines.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 7bea93fbc7e..6d506f1d3d5 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -167,10 +167,16 @@ jobs: # run tests (exclude pytest.AMReX when running Python tests) ctest --test-dir build --output-on-failure -E AMReX + ctest_exit_status=$? - # find and print backtraces + # find and print backtraces (regardless of CTest exit status) find build/bin/ -type f -name "Backtrace*" \ -exec echo -e "\nBacktrace\n---------\n{}\n---------" \; \ -exec cat {} \; \ -exec echo -e "\n" \; + + # exit program based on CTest exit status + if (( ${ctest_exit_status} != 0 )); then + exit ${ctest_exit_status} + fi displayName: 'Test' From 60e23161f33f3c4176dd67847fe0a76e600349e1 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni <59625522+EZoni@users.noreply.github.com> Date: Tue, 29 Oct 2024 13:45:33 -0700 Subject: [PATCH 4/7] Inject more NaN to trigger crash and backtrace --- .../PsatdAlgorithmJConstantInTime.cpp | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp index 991bf981924..78422f8a408 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp @@ -845,18 +845,18 @@ PsatdAlgorithmJConstantInTime::VayDeposition (SpectralFieldData& field_data) #endif // Compute Jx - if (kx_mod != 0._rt) { fields(i,j,k,Idx.Jx_mid) = I * Dx / kx_mod; } - else { fields(i,j,k,Idx.Jx_mid) = 0._rt; } + if (kx_mod != 0._rt) { fields(i,j,k,Idx.Jx_mid) = std::nan(""); } //I * Dx / kx_mod; } + else { fields(i,j,k,Idx.Jx_mid) = std::nan(""); } //0._rt; } #if defined(WARPX_DIM_3D) // Compute Jy - if (ky_mod != 0._rt) { fields(i,j,k,Idx.Jy_mid) = I * Dy / ky_mod; } - else { fields(i,j,k,Idx.Jy_mid) = 0._rt; } + if (ky_mod != 0._rt) { fields(i,j,k,Idx.Jy_mid) = std::nan(""); } //I * Dy / ky_mod; } + else { fields(i,j,k,Idx.Jy_mid) = std::nan(""); } //0._rt; } #endif // Compute Jz - if (kz_mod != 0._rt) { fields(i,j,k,Idx.Jz_mid) = I * Dz / kz_mod; } - else { fields(i,j,k,Idx.Jz_mid) = 0._rt; } + if (kz_mod != 0._rt) { fields(i,j,k,Idx.Jz_mid) = std::nan(""); } //I * Dz / kz_mod; } + else { fields(i,j,k,Idx.Jz_mid) = std::nan(""); } //0._rt; } }); } } From e761ef12dbff6c5e3536cfb0054482c422a6cd76 Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Tue, 29 Oct 2024 15:18:00 -0700 Subject: [PATCH 5/7] Handle CTest exit status correctly --- .azure-pipelines.yml | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) diff --git a/.azure-pipelines.yml b/.azure-pipelines.yml index 6d506f1d3d5..1d5127ae5a1 100644 --- a/.azure-pipelines.yml +++ b/.azure-pipelines.yml @@ -77,7 +77,7 @@ jobs: displayName: Cache Python Libraries - bash: | - set -eu -o pipefail + set -o nounset errexit pipefail cat /proc/cpuinfo | grep "model name" | sort -u df -h echo 'Acquire::Retries "3";' | sudo tee /etc/apt/apt.conf.d/80-retries @@ -146,9 +146,10 @@ jobs: displayName: 'Install dependencies' - bash: | - set -eu -o pipefail + # set options + set -o nounset errexit pipefail + # display disk space usage df -h - # configure export AMReX_CMAKE_FLAGS="-DAMReX_ASSERTIONS=ON -DAMReX_TESTING=ON" cmake -S . -B build \ @@ -156,27 +157,25 @@ jobs: ${WARPX_CMAKE_FLAGS} \ -DWarpX_TEST_CLEANUP=ON \ -DWarpX_TEST_FPETRAP=ON - # build cmake --build build -j 2 + # display disk space usage df -h displayName: 'Build' - bash: | - set -eu -o pipefail - + # set options + set -o nounset errexit pipefail # run tests (exclude pytest.AMReX when running Python tests) ctest --test-dir build --output-on-failure -E AMReX - ctest_exit_status=$? + displayName: 'Test' - # find and print backtraces (regardless of CTest exit status) + - bash: | + # set options + set -o nounset errexit pipefail + # find and print backtrace find build/bin/ -type f -name "Backtrace*" \ - -exec echo -e "\nBacktrace\n---------\n{}\n---------" \; \ - -exec cat {} \; \ - -exec echo -e "\n" \; - - # exit program based on CTest exit status - if (( ${ctest_exit_status} != 0 )); then - exit ${ctest_exit_status} - fi - displayName: 'Test' + -exec echo -e "\nBacktrace\n---------\n{}\n---------" \; \ + -exec cat {} \; + displayName: 'Logs' + condition: always() From aa96e8b626fe1094e8f0baeb56ec8c34ee50b23c Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 1 Nov 2024 16:45:06 -0700 Subject: [PATCH 6/7] Do not remove backtrace files in `cleanup` test --- Examples/CMakeLists.txt | 2 +- Examples/test_cleanup.cmake | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 Examples/test_cleanup.cmake diff --git a/Examples/CMakeLists.txt b/Examples/CMakeLists.txt index f36bcbb9973..728c2142932 100644 --- a/Examples/CMakeLists.txt +++ b/Examples/CMakeLists.txt @@ -198,7 +198,7 @@ function(add_warpx_test if(WarpX_TEST_CLEANUP) add_test( NAME ${name}.cleanup - COMMAND ${CMAKE_COMMAND} -E rm -rf ${THIS_WORKING_DIR} + COMMAND ${CMAKE_COMMAND} -P ${CMAKE_SOURCE_DIR}/Examples/test_cleanup.cmake ${THIS_WORKING_DIR} ) # test cleanup depends on test run set_property(TEST ${name}.cleanup APPEND PROPERTY DEPENDS "${name}.run") diff --git a/Examples/test_cleanup.cmake b/Examples/test_cleanup.cmake new file mode 100644 index 00000000000..b15e31e1f5d --- /dev/null +++ b/Examples/test_cleanup.cmake @@ -0,0 +1,7 @@ +# delete all test files except backtrace +file(GLOB test_files ${CMAKE_ARGV3}/*) +foreach(file ${test_files}) + if(NOT ${file} MATCHES "Backtrace*") + execute_process(COMMAND ${CMAKE_COMMAND} -E rm -r ${file}) + endif() +endforeach() From 28fb849ba13eef662cc9e0c18cda3d6bfd6b05eb Mon Sep 17 00:00:00 2001 From: Edoardo Zoni Date: Fri, 1 Nov 2024 18:12:27 -0700 Subject: [PATCH 7/7] Remove NaN and restore correct code behavior --- .../PsatdAlgorithmComoving.cpp | 24 +++++++++---------- .../PsatdAlgorithmJConstantInTime.cpp | 12 +++++----- 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp index ae6e007053a..f809c3f2c20 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmComoving.cpp @@ -138,24 +138,24 @@ PsatdAlgorithmComoving::pushSpectralFields (SpectralFieldData& f) const const Complex X4 = X4_arr(i,j,k); // Update E - fields(i,j,k,Idx.Ex) = std::nan(""); //C*Ex_old + S_ck*c2*I*(ky_mod*Bz_old - kz_mod*By_old) - //+ X4*Jx - I*(X2*rho_new - X3*rho_old)*kx_mod; + fields(i,j,k,Idx.Ex) = C*Ex_old + S_ck*c2*I*(ky_mod*Bz_old - kz_mod*By_old) + + X4*Jx - I*(X2*rho_new - X3*rho_old)*kx_mod; - fields(i,j,k,Idx.Ey) = std::nan(""); //C*Ey_old + S_ck*c2*I*(kz_mod*Bx_old - kx_mod*Bz_old) - //+ X4*Jy - I*(X2*rho_new - X3*rho_old)*ky_mod; + fields(i,j,k,Idx.Ey) = C*Ey_old + S_ck*c2*I*(kz_mod*Bx_old - kx_mod*Bz_old) + + X4*Jy - I*(X2*rho_new - X3*rho_old)*ky_mod; - fields(i,j,k,Idx.Ez) = std::nan(""); //C*Ez_old + S_ck*c2*I*(kx_mod*By_old - ky_mod*Bx_old) - //+ X4*Jz - I*(X2*rho_new - X3*rho_old)*kz_mod; + fields(i,j,k,Idx.Ez) = C*Ez_old + S_ck*c2*I*(kx_mod*By_old - ky_mod*Bx_old) + + X4*Jz - I*(X2*rho_new - X3*rho_old)*kz_mod; // Update B - fields(i,j,k,Idx.Bx) = std::nan(""); //C*Bx_old - S_ck*I*(ky_mod*Ez_old - kz_mod*Ey_old) - //+ X1*I*(ky_mod*Jz - kz_mod*Jy); + fields(i,j,k,Idx.Bx) = C*Bx_old - S_ck*I*(ky_mod*Ez_old - kz_mod*Ey_old) + + X1*I*(ky_mod*Jz - kz_mod*Jy); - fields(i,j,k,Idx.By) = std::nan(""); //C*By_old - S_ck*I*(kz_mod*Ex_old - kx_mod*Ez_old) - //+ X1*I*(kz_mod*Jx - kx_mod*Jz); + fields(i,j,k,Idx.By) = C*By_old - S_ck*I*(kz_mod*Ex_old - kx_mod*Ez_old) + + X1*I*(kz_mod*Jx - kx_mod*Jz); - fields(i,j,k,Idx.Bz) = std::nan(""); //C*Bz_old - S_ck*I*(kx_mod*Ey_old - ky_mod*Ex_old) - //+ X1*I*(kx_mod*Jy - ky_mod*Jx); + fields(i,j,k,Idx.Bz) = C*Bz_old - S_ck*I*(kx_mod*Ey_old - ky_mod*Ex_old) + + X1*I*(kx_mod*Jy - ky_mod*Jx); }); } } diff --git a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp index 78422f8a408..991bf981924 100644 --- a/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp +++ b/Source/FieldSolver/SpectralSolver/SpectralAlgorithms/PsatdAlgorithmJConstantInTime.cpp @@ -845,18 +845,18 @@ PsatdAlgorithmJConstantInTime::VayDeposition (SpectralFieldData& field_data) #endif // Compute Jx - if (kx_mod != 0._rt) { fields(i,j,k,Idx.Jx_mid) = std::nan(""); } //I * Dx / kx_mod; } - else { fields(i,j,k,Idx.Jx_mid) = std::nan(""); } //0._rt; } + if (kx_mod != 0._rt) { fields(i,j,k,Idx.Jx_mid) = I * Dx / kx_mod; } + else { fields(i,j,k,Idx.Jx_mid) = 0._rt; } #if defined(WARPX_DIM_3D) // Compute Jy - if (ky_mod != 0._rt) { fields(i,j,k,Idx.Jy_mid) = std::nan(""); } //I * Dy / ky_mod; } - else { fields(i,j,k,Idx.Jy_mid) = std::nan(""); } //0._rt; } + if (ky_mod != 0._rt) { fields(i,j,k,Idx.Jy_mid) = I * Dy / ky_mod; } + else { fields(i,j,k,Idx.Jy_mid) = 0._rt; } #endif // Compute Jz - if (kz_mod != 0._rt) { fields(i,j,k,Idx.Jz_mid) = std::nan(""); } //I * Dz / kz_mod; } - else { fields(i,j,k,Idx.Jz_mid) = std::nan(""); } //0._rt; } + if (kz_mod != 0._rt) { fields(i,j,k,Idx.Jz_mid) = I * Dz / kz_mod; } + else { fields(i,j,k,Idx.Jz_mid) = 0._rt; } }); } }