From 05d9f80029e21819cb9ec35a5bf56f4dfa453504 Mon Sep 17 00:00:00 2001 From: Travis Downs Date: Tue, 1 Apr 2025 13:25:31 -0300 Subject: [PATCH] Print incrementally in sigsegv handler If we crash due to segfault, we want to print the backtrace incrementally, frame-by-frame, rather than all at once. This is because the backtrace call may crash at some problematic frame, in which case we get no output at all: these are very hard to diagnose. Instead, introduce an "immediate" mode to backtrace buffer which prints all appended strings immediately, and add an incremental mode to the backtrace function which calls ::backtrace repeatedly, increasing the number of requested frames by 1 each time. If we crash at some frame N we will have printed all frames 1..N-1 already. Before this change the backtrace looks like this using the reproducer in scylladb/seastar#2697: ``` Backtrace: ``` After it looks like: ``` Backtrace: /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x36ce6a /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x32fbba /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x342d0b /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x35cf6c /lib/x86_64-linux-gnu/libc.so.6+0x4251f /lib/x86_64-linux-gnu/libgcc_s.so.1+0x17585 /lib/x86_64-linux-gnu/libc.so.6+0x133bb2 /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x36d0d7 /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x32fb84 /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x32f0e7 /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x330281 /lib/x86_64-linux-gnu/libc.so.6+0x4251f /lib/x86_64-linux-gnu/libgcc_s.so.1+0x14789 /lib/x86_64-linux-gnu/libgcc_s.so.1+0x16f54 /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x2b98a8 /lib/x86_64-linux-gnu/libgcc_s.so.1+0x173b8 ``` Fixes #2710. --- include/seastar/util/backtrace.hh | 30 ++++++++++++++++++---- src/core/reactor.cc | 41 +++++++++++++++++++++++-------- 2 files changed, 56 insertions(+), 15 deletions(-) diff --git a/include/seastar/util/backtrace.hh b/include/seastar/util/backtrace.hh index 0676004a440..3f6ab1bdf95 100644 --- a/include/seastar/util/backtrace.hh +++ b/include/seastar/util/backtrace.hh @@ -61,16 +61,36 @@ bool operator==(const frame& a, const frame& b) noexcept; frame decorate(uintptr_t addr) noexcept; // Invokes func for each frame passing it as argument. +// incremental=false is the default mode and simply calls ::backtrace once +// and then calls func for each frame. If the ::backtrace call crashes, +// func will not be called. +// incremental=true will call ::backtrace in a loop, increasing the number of +// frames to capture by one each time. This is useful for cases where +// ::backtrace itself may crash, e.g., in a crash handler, in this case the +// func will be called for each frame which didn't crash. See #2710. This +// is much slower, however. SEASTAR_MODULE_EXPORT template -void backtrace(Func&& func) noexcept(noexcept(func(frame()))) { +void backtrace(Func&& func, bool incremental = false) noexcept(noexcept(func(frame()))) { #ifdef HAVE_EXECINFO constexpr size_t max_backtrace = 100; void* buffer[max_backtrace]; - int n = ::backtrace(buffer, max_backtrace); - for (int i = 0; i < n; ++i) { - auto ip = reinterpret_cast(buffer[i]); - func(decorate(ip - 1)); + + if (incremental) { + for (size_t last_frame = 1; last_frame <= max_backtrace; ++last_frame) { + int n = ::backtrace(buffer, last_frame); + if (n < static_cast(last_frame)) { + return; + } + auto ip = reinterpret_cast(buffer[last_frame-1]); + func(decorate(ip - 1)); + } + } else { + int n = ::backtrace(buffer, max_backtrace); + for (int i = 0; i < n; ++i) { + auto ip = reinterpret_cast(buffer[i]); + func(decorate(ip - 1)); + } } #else // Not implemented yet diff --git a/src/core/reactor.cc b/src/core/reactor.cc index d68f86175bb..2d23131c80c 100644 --- a/src/core/reactor.cc +++ b/src/core/reactor.cc @@ -766,8 +766,11 @@ class backtrace_buffer { static constexpr unsigned _max_size = 8 << 10; unsigned _pos = 0; char _buf[_max_size]; + bool _immediate; public: - backtrace_buffer() = default; + backtrace_buffer(bool immediate = false) + : _immediate{immediate} {} + ~backtrace_buffer() { flush(); } @@ -779,13 +782,16 @@ class backtrace_buffer { backtrace_buffer &operator = (backtrace_buffer &&) = delete; void flush() noexcept { - if (_pos > 0) { + if (!_immediate && _pos > 0) { print_safe(_buf, _pos); _pos = 0; } } void reserve(size_t len) noexcept { + if (_immediate) { + return; + } SEASTAR_ASSERT(len < _max_size); if (_pos + len >= _max_size) { flush(); @@ -793,9 +799,13 @@ class backtrace_buffer { } void append(const char* str, size_t len) noexcept { - reserve(len); - memcpy(_buf + _pos, str, len); - _pos += len; + if (_immediate) { + print_safe(str, len); + } else { + reserve(len); + memcpy(_buf + _pos, str, len); + _pos += len; + } } void append(const char* str) noexcept { append(str, strlen(str)); } @@ -825,7 +835,7 @@ class backtrace_buffer { append("0x"); append_hex(f.addr); append("\n"); - }); + }, _immediate); } void append_backtrace_oneline() noexcept { @@ -833,7 +843,7 @@ class backtrace_buffer { reserve(3 + sizeof(f.addr) * 2); append(" 0x"); append_hex(f.addr); - }); + }, _immediate); } }; @@ -856,8 +866,17 @@ static void print_with_backtrace(backtrace_buffer& buf, bool oneline) noexcept { } } -static void print_with_backtrace(const char* cause, bool oneline = false) noexcept { - backtrace_buffer buf; +// Print the current backtrace to stdout with the given cause. +// If oneline is true, backtrace is printed entirely on one line, +// otherwise it is printed with 1 line per frame. +// If immediate is true, the backtrace is printed frame by frame +// with a call to write(2), otherwise it is printed in a single +// call to write(2). The former strategy is more robust in cases +// where the backtrace itself may crash at some point down the stack +// while the latter is more efficient and avoids splitting output +// in the face of concurrent logging by other shards. +static void print_with_backtrace(const char* cause, bool oneline = false, bool immediate = false) noexcept { + backtrace_buffer buf(immediate); buf.append(cause); print_with_backtrace(buf, oneline); } @@ -4053,7 +4072,9 @@ static void sigsegv_action(siginfo_t *info, ucontext_t* uc) noexcept { print_zero_padded_hex_safe(f.so->end - f.so->begin); print_safe("]\n"); - print_with_backtrace("Segmentation fault"); + // print the backtrace in immediate mode, so if we crash + // during the backtrace we get as much output as possible + print_with_backtrace("Segmentation fault", false, true); reraise_signal(SIGSEGV); }