Print incrementally in sigsegv handler

include/seastar/util/backtrace.hh

@@ -61,16 +61,36 @@ bool operator==(const frame& a, const frame& b) noexcept;
 frame decorate(uintptr_t addr) noexcept;
 
 // Invokes func for each frame passing it as argument.
+// incremental=false is the default mode and simply calls ::backtrace once
+// and then calls func for each frame. If the ::backtrace call crashes,
+// func will not be called.
+// incremental=true will call ::backtrace in a loop, increasing the number of
+// frames to capture by one each time. This is useful for cases where
+// ::backtrace itself may crash, e.g., in a crash handler, in this case the
+// func will be called for each frame which didn't crash. See #2710. This
+// is much slower, however.
 SEASTAR_MODULE_EXPORT
 template<typename Func>
-void backtrace(Func&& func) noexcept(noexcept(func(frame()))) {
+void backtrace(Func&& func, bool incremental = false) noexcept(noexcept(func(frame()))) {
 #ifdef HAVE_EXECINFO
     constexpr size_t max_backtrace = 100;
     void* buffer[max_backtrace];
-    int n = ::backtrace(buffer, max_backtrace);
-    for (int i = 0; i < n; ++i) {
-        auto ip = reinterpret_cast<uintptr_t>(buffer[i]);
-        func(decorate(ip - 1));
+
+    if (incremental) {
+        for (size_t last_frame = 1; last_frame <= max_backtrace; ++last_frame) {
+            int n = ::backtrace(buffer, last_frame);
+            if (n < static_cast<int>(last_frame)) {
+                return;
+            }
+            auto ip = reinterpret_cast<uintptr_t>(buffer[last_frame-1]);
+            func(decorate(ip - 1));
+        }
+    } else {
+        int n = ::backtrace(buffer, max_backtrace);
+        for (int i = 0; i < n; ++i) {
+            auto ip = reinterpret_cast<uintptr_t>(buffer[i]);
+            func(decorate(ip - 1));
+        }
     }
 #else
 // Not implemented yet

src/core/reactor.cc

@@ -766,8 +766,11 @@ class backtrace_buffer {
     static constexpr unsigned _max_size = 8 << 10;
     unsigned _pos = 0;
     char _buf[_max_size];
+    bool _immediate;
 public:
-    backtrace_buffer() = default;
+    backtrace_buffer(bool immediate = false)
+        : _immediate{immediate} {}
+
     ~backtrace_buffer() {
         flush();
     }
@@ -779,23 +782,30 @@ class backtrace_buffer {
     backtrace_buffer &operator = (backtrace_buffer &&) = delete;
 
     void flush() noexcept {
-        if (_pos > 0) {
+        if (!_immediate && _pos > 0) {
             print_safe(_buf, _pos);
             _pos = 0;
         }
     }
 
     void reserve(size_t len) noexcept {
+        if (_immediate) {
+            return;
+        }
         SEASTAR_ASSERT(len < _max_size);
         if (_pos + len >= _max_size) {
             flush();
         }
     }
 
     void append(const char* str, size_t len) noexcept {
-        reserve(len);
-        memcpy(_buf + _pos, str, len);
-        _pos += len;
+        if (_immediate) {
+            print_safe(str, len);
+        } else {
+            reserve(len);
+            memcpy(_buf + _pos, str, len);
+            _pos += len;
+        }
     }
 
     void append(const char* str) noexcept { append(str, strlen(str)); }
@@ -825,15 +835,15 @@ class backtrace_buffer {
             append("0x");
             append_hex(f.addr);
             append("\n");
-        });
+        }, _immediate);
     }
 
     void append_backtrace_oneline() noexcept {
         backtrace([this] (frame f) noexcept {
             reserve(3 + sizeof(f.addr) * 2);
             append(" 0x");
             append_hex(f.addr);
-        });
+        }, _immediate);
     }
 };
 
@@ -856,8 +866,17 @@ static void print_with_backtrace(backtrace_buffer& buf, bool oneline) noexcept {
   }
 }
 
-static void print_with_backtrace(const char* cause, bool oneline = false) noexcept {
-    backtrace_buffer buf;
+// Print the current backtrace to stdout with the given cause.
+// If oneline is true, backtrace is printed entirely on one line,
+// otherwise it is printed with 1 line per frame.
+// If immediate is true, the backtrace is printed frame by frame
+// with a call to write(2), otherwise it is printed in a single
+// call to write(2). The former strategy is more robust in cases
+// where the backtrace itself may crash at some point down the stack
+// while the latter is more efficient and avoids splitting output
+// in the face of concurrent logging by other shards.
+static void print_with_backtrace(const char* cause, bool oneline = false, bool immediate = false) noexcept {
+    backtrace_buffer buf(immediate);
     buf.append(cause);
     print_with_backtrace(buf, oneline);
 }
@@ -4053,7 +4072,9 @@ static void sigsegv_action(siginfo_t *info, ucontext_t* uc) noexcept {
     print_zero_padded_hex_safe(f.so->end - f.so->begin);
     print_safe("]\n");
 
-    print_with_backtrace("Segmentation fault");
+    // print the backtrace in immediate mode, so if we crash
+    // during the backtrace we get as much output as possible
+    print_with_backtrace("Segmentation fault", false, true);
     reraise_signal(SIGSEGV);
 }