Print incrementally in sigsegv handler

If we crash due to segfault, we want to print the backtrace
incrementally, frame-by-frame, rather than all at once. This is because
the backtrace call may crash at some problematic frame, in which case we
get no output at all: these are very hard to diagnose.

Instead, introduce an "immediate" mode to backtrace buffer which prints
all appended strings immediately, and add an incremental mode to the
backtrace function which calls ::backtrace repeatedly, increasing the
number of requested frames by 1 each time. If we crash at some frame N
we will have printed all frames 1..N-1 already.

Before this change the backtrace looks like this using the reproducer
in #2697:

```
Backtrace:
```

After it looks like:

```
Backtrace:
  /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x36ce6a
  /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x32fbba
  /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x342d0b
  /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x35cf6c
  /lib/x86_64-linux-gnu/libc.so.6+0x4251f
  /lib/x86_64-linux-gnu/libgcc_s.so.1+0x17585
  /lib/x86_64-linux-gnu/libc.so.6+0x133bb2
  /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x36d0d7
  /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x32fb84
  /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x32f0e7
  /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x330281
  /lib/x86_64-linux-gnu/libc.so.6+0x4251f
  /lib/x86_64-linux-gnu/libgcc_s.so.1+0x14789
  /lib/x86_64-linux-gnu/libgcc_s.so.1+0x16f54
  /home/tdowns/dev/seastar/build-clang-18/dev/libseastar.so+0x2b98a8
  /lib/x86_64-linux-gnu/libgcc_s.so.1+0x173b8
```

Fixes #2710.

Author: travisdowns

Diff for: include/seastar/util/backtrace.hh

@@ -61,16 +61,36 @@ bool operator==(const frame& a, const frame& b) noexcept;
 frame decorate(uintptr_t addr) noexcept;
 
 // Invokes func for each frame passing it as argument.
+// incremental=false is the default mode and simply calls ::backtrace once
+// and then calls func for each frame. If the ::backtrace call crashes,
+// func will not be called.
+// incremental=true will call ::backtrace in a loop, increasing the number of
+// frames to capture by one each time. This is useful for cases where
+// ::backtrace itself may crash, e.g., in a crash handler, in this case the
+// func will be called for each frame which didn't crash. See #2710. This
+// is much slower, however.
 SEASTAR_MODULE_EXPORT
 template<typename Func>
-void backtrace(Func&& func) noexcept(noexcept(func(frame()))) {
+void backtrace(Func&& func, bool incremental = false) noexcept(noexcept(func(frame()))) {
 #ifdef HAVE_EXECINFO
     constexpr size_t max_backtrace = 100;
     void* buffer[max_backtrace];
-    int n = ::backtrace(buffer, max_backtrace);
-    for (int i = 0; i < n; ++i) {
-        auto ip = reinterpret_cast<uintptr_t>(buffer[i]);
-        func(decorate(ip - 1));
+
+    if (incremental) {
+        for (size_t last_frame = 1; last_frame <= max_backtrace; ++last_frame) {
+            int n = ::backtrace(buffer, last_frame);
+            if (n < static_cast<int>(last_frame)) {
+                return;
+            }
+            auto ip = reinterpret_cast<uintptr_t>(buffer[last_frame-1]);
+            func(decorate(ip - 1));
+        }
+    } else {
+        int n = ::backtrace(buffer, max_backtrace);
+        for (int i = 0; i < n; ++i) {
+            auto ip = reinterpret_cast<uintptr_t>(buffer[i]);
+            func(decorate(ip - 1));
+        }
     }
 #else
 // Not implemented yet

Diff for: src/core/reactor.cc

@@ -766,8 +766,11 @@ class backtrace_buffer {
     static constexpr unsigned _max_size = 8 << 10;
     unsigned _pos = 0;
     char _buf[_max_size];
+    bool _immediate;
 public:
-    backtrace_buffer() = default;
+    backtrace_buffer(bool immediate = false)
+        : _immediate{immediate} {}
+
     ~backtrace_buffer() {
         flush();
     }
@@ -779,23 +782,30 @@ class backtrace_buffer {
     backtrace_buffer &operator = (backtrace_buffer &&) = delete;
 
     void flush() noexcept {
-        if (_pos > 0) {
+        if (!_immediate && _pos > 0) {
             print_safe(_buf, _pos);
             _pos = 0;
         }
     }
 
     void reserve(size_t len) noexcept {
+        if (_immediate) {
+            return;
+        }
         SEASTAR_ASSERT(len < _max_size);
         if (_pos + len >= _max_size) {
             flush();
         }
     }
 
     void append(const char* str, size_t len) noexcept {
-        reserve(len);
-        memcpy(_buf + _pos, str, len);
-        _pos += len;
+        if (_immediate) {
+            print_safe(str, len);
+        } else {
+            reserve(len);
+            memcpy(_buf + _pos, str, len);
+            _pos += len;
+        }
     }
 
     void append(const char* str) noexcept { append(str, strlen(str)); }
@@ -825,15 +835,15 @@ class backtrace_buffer {
             append("0x");
             append_hex(f.addr);
             append("\n");
-        });
+        }, _immediate);
     }
 
     void append_backtrace_oneline() noexcept {
         backtrace([this] (frame f) noexcept {
             reserve(3 + sizeof(f.addr) * 2);
             append(" 0x");
             append_hex(f.addr);
-        });
+        }, _immediate);
     }
 };
 
@@ -856,8 +866,17 @@ static void print_with_backtrace(backtrace_buffer& buf, bool oneline) noexcept {
   }
 }
 
-static void print_with_backtrace(const char* cause, bool oneline = false) noexcept {
-    backtrace_buffer buf;
+// Print the current backtrace to stdout with the given cause.
+// If oneline is true, backtrace is printed entirely on one line,
+// otherwise it is printed with 1 line per frame.
+// If immediate is true, the backtrace is printed frame by frame
+// with a call to write(2), otherwise it is printed in a single
+// call to write(2). The former strategy is more robust in cases
+// where the backtrace itself may crash at some point down the stack
+// while the latter is more efficient and avoids splitting output
+// in the face of concurrent logging by other shards.
+static void print_with_backtrace(const char* cause, bool oneline = false, bool immediate = false) noexcept {
+    backtrace_buffer buf(immediate);
     buf.append(cause);
     print_with_backtrace(buf, oneline);
 }
@@ -4053,7 +4072,9 @@ static void sigsegv_action(siginfo_t *info, ucontext_t* uc) noexcept {
     print_zero_padded_hex_safe(f.so->end - f.so->begin);
     print_safe("]\n");
 
-    print_with_backtrace("Segmentation fault");
+    // print the backtrace in immediate mode, so if we crash
+    // during the backtrace we get as much output as possible
+    print_with_backtrace("Segmentation fault", false, true);
     reraise_signal(SIGSEGV);
 }