android: capture JIT-thread SIGSEGV by chaining handler on top of Mono

Agent-Logs-Url: https://github.com/winnerspiros/osu/sessions/b1f176a6-7fdf-42cc-86ea-a90cf728ffa2

Co-authored-by: winnerspiros <1675249+winnerspiros@users.noreply.github.com>

Author: Copilot

osu.Android/CrashDiagnostics.cs

@@ -15,13 +15,14 @@ namespace osu.Android
     /// <summary>
     /// Centralised Android crash-diagnostics plumbing.
     ///
-    /// We write everything to <b>internal</b> app storage (<c>FilesDir</c>) because external
-    /// storage is FUSE-backed, scoped-storage-restricted, and may not be ready at the
-    /// instant a very-early crash hits. On the next normal startup we mirror the internal
-    /// crash log to <c>GetExternalFilesDir(null)</c> so the user can grab it via the Files
-    /// app on an unrooted device, then truncate the internal copy.
+    /// We write everything to <b>both</b> internal app storage (<c>FilesDir</c>) and external
+    /// app storage (<c>GetExternalFilesDir(null)</c>) when both are available. Internal is the
+    /// reliable target for the very-early window where external storage may not yet be ready;
+    /// external is reachable by the user via the Files app on an unrooted device and receives
+    /// alive markers / managed-exception dumps in real time so the user does not have to wait
+    /// for a successful next startup to mirror the data over.
     ///
-    /// Files (all relative to <c>FilesDir</c>):
+    /// Files (relative to each storage dir):
     ///   <list type="bullet">
     ///     <item><c>native_crash.log</c> — append target for both the native handler and the managed last-chance hooks; also receives "I am alive" startup markers.</item>
     ///     <item><c>crash_handler_installed.txt</c> — sentinel dropped immediately after <c>nInstallCrashHandler</c> returns. Lets us distinguish "handler never installed (P/Invoke failed → libosu_native.so missing)" from "handler installed but signal bypassed it".</item>
@@ -37,6 +38,9 @@ internal static class CrashDiagnostics
 
         private static string? internalDir;
         private static string? externalDir;
+        private static string? sentinelPath;
+        private static string? installedLogPath;
+        private static bool sentinelWritten;
 
         /// <summary>
         /// Installs the native crash handler against the internal-storage log path, drops the
@@ -51,22 +55,24 @@ public static void InstallNativeHandler(Context context)
             {
                 resolveDirs(context);
 
-                string? logPath = internalDir != null ? Path.Combine(internalDir, CRASH_LOG_NAME) : null;
+                installedLogPath = internalDir != null ? Path.Combine(internalDir, CRASH_LOG_NAME) : null;
 
                 // The native handler is best-effort. Wrap so a DllNotFoundException
                 // (libosu_native.so missing from the APK) cannot itself crash us.
                 try
                 {
-                    OboeAudioBridge.nInstallCrashHandler(logPath);
+                    OboeAudioBridge.nInstallCrashHandler(installedLogPath);
 
                     // Sentinel: only written when nInstallCrashHandler returned without throwing.
                     if (internalDir != null)
                     {
                         try
                         {
+                            sentinelPath = Path.Combine(internalDir, SENTINEL_NAME);
                             File.WriteAllText(
-                                Path.Combine(internalDir, SENTINEL_NAME),
-                                $"installed_at={DateTime.UtcNow:O}\nlog_path={logPath ?? "<none>"}\n");
+                                sentinelPath,
+                                $"installed_at={DateTime.UtcNow:O}\nlog_path={installedLogPath ?? "<none>"}\n");
+                            sentinelWritten = true;
                         }
                         catch (Exception e) { Debug.WriteLine($"[osu!] Could not write crash-handler sentinel: {e.Message}"); }
                     }
@@ -87,31 +93,38 @@ public static void InstallNativeHandler(Context context)
         }
 
         /// <summary>
-        /// Append a single-line "I am alive" marker to the internal crash log so that, when
-        /// we later inspect a truncated/empty file after a crash, the last-written marker
-        /// pinpoints which startup phase died.
+        /// Re-install the native signal handlers from a later startup phase, after the Mono
+        /// runtime has installed its own SIGSEGV handler. This is what actually lets us catch
+        /// JIT-thread null-deref crashes — without it, Mono's handler intercepts the fault
+        /// first and re-raises via <c>tgkill</c> (visible in tombstones as
+        /// <c>si_code = SI_TKILL</c>) without ever forwarding to us.
         /// </summary>
-        public static void WriteAliveMarker(string phase)
+        public static void ReinstallNativeHandler()
         {
             try
             {
-                if (internalDir == null) return;
-
-                string path = Path.Combine(internalDir, CRASH_LOG_NAME);
-                string line = $"=== ALIVE [{DateTime.UtcNow:O}] {phase} ===\n";
-
-                // Append using a bounded write — never throw, never block.
-                using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite);
-                using var sw = new StreamWriter(fs);
-                sw.Write(line);
-                sw.Flush();
+                OboeAudioBridge.nReinstallCrashHandler();
+                WriteAliveMarker("CrashDiagnostics.ReinstallNativeHandler (chained on top of Mono)");
             }
             catch (Exception e)
             {
-                Debug.WriteLine($"[osu!] WriteAliveMarker({phase}) failed: {e.Message}");
+                Debug.WriteLine($"[osu!] nReinstallCrashHandler P/Invoke failed: {e.Message}");
             }
         }
 
+        /// <summary>
+        /// Append a single-line "I am alive" marker to the crash log so that, when we later
+        /// inspect a truncated/empty file after a crash, the last-written marker pinpoints
+        /// which startup phase died. Writes to both internal and external storage so the user
+        /// can pull the file immediately without waiting for a successful next startup to
+        /// mirror it over.
+        /// </summary>
+        public static void WriteAliveMarker(string phase)
+        {
+            string line = $"=== ALIVE [{DateTime.UtcNow:O}] {phase} ===\n";
+            appendToBoth(line);
+        }
+
         /// <summary>
         /// One-shot post-crash mirror: if an internal <c>native_crash.log</c> exists and is
         /// non-empty, copy it to external app storage (so the user can pull it via the Files
@@ -179,34 +192,114 @@ public static void InstallManagedExceptionHooks()
                 writeManagedException("TaskScheduler.UnobservedTaskException", e.Exception);
                 // Don't mark observed — the framework / sentry pipeline still wants to see it.
             };
+
+            // FirstChanceException fires for *every* managed exception, even ones that get
+            // caught later. On non-main managed threads (e.g. the Draw thread), Mono on
+            // Android does not always route an unhandled exception through
+            // AppDomain.UnhandledException before aborting — so without this hook the
+            // exception that ultimately kills the process can vanish without trace. We
+            // record it here on every throw so the *last* recorded exception before a
+            // SIGSEGV/SIGABRT is the candidate culprit. To avoid drowning the log in noise
+            // we filter by exception type — only fatal-ish kinds are recorded.
+            try
+            {
+                AppDomain.CurrentDomain.FirstChanceException += (_, e) =>
+                {
+                    if (e.Exception is NullReferenceException
+                        or AccessViolationException
+                        or StackOverflowException
+                        or TypeInitializationException
+                        or DllNotFoundException
+                        or EntryPointNotFoundException
+                        or BadImageFormatException
+                        or TypeLoadException
+                        or MissingMethodException
+                        or MissingFieldException
+                        or InvalidProgramException)
+                    {
+                        writeManagedException($"FirstChanceException ({e.Exception.GetType().Name})", e.Exception);
+                    }
+                };
+            }
+            catch (Exception e)
+            {
+                Debug.WriteLine($"[osu!] Could not install FirstChanceException hook: {e.Message}");
+            }
+        }
+
+        /// <summary>
+        /// Records a one-line summary of the native handler install state (sentinel exists?
+        /// log path?) so the very first thing we see in the log on the next inspection tells
+        /// us whether the native handler is even in place.
+        /// </summary>
+        public static void WriteInstallState()
+        {
+            try
+            {
+                string sentinelState;
+
+                if (sentinelWritten && sentinelPath != null && File.Exists(sentinelPath))
+                    sentinelState = "present";
+                else if (sentinelWritten)
+                    sentinelState = "written-but-missing";
+                else
+                    sentinelState = "absent";
+
+                appendToBoth($"=== INSTALL_STATE sentinel={sentinelState} log_path={installedLogPath ?? "<none>"} internal_dir={internalDir ?? "<none>"} external_dir={externalDir ?? "<none>"} ===\n");
+            }
+            catch (Exception e)
+            {
+                Debug.WriteLine($"[osu!] WriteInstallState failed: {e.Message}");
+            }
         }
 
         private static void writeManagedException(string source, Exception? ex)
         {
             try
             {
-                if (internalDir == null) return;
+                string block =
+                    "\n=========================================================\n" +
+                    "=== MANAGED EXCEPTION ===\n" +
+                    $"  source     = {source}\n" +
+                    $"  utc_time   = {DateTime.UtcNow:O}\n" +
+                    $"  thread_id  = {Environment.CurrentManagedThreadId}\n" +
+                    $"  thread_name= {Thread.CurrentThread.Name ?? "<null>"}\n" +
+                    "\n" +
+                    (ex?.ToString() ?? "<no exception object>") + "\n" +
+                    "=== END OF MANAGED EXCEPTION ===\n\n";
+                appendToBoth(block);
+            }
+            catch (Exception e)
+            {
+                Debug.WriteLine($"[osu!] writeManagedException failed: {e.Message}");
+            }
+        }
+
+        // Append the same payload to both internal (FilesDir) and external (GetExternalFilesDir)
+        // crash logs. Either may legitimately be unavailable; failure of one path must not
+        // prevent the other from being written. Each write is bounded, non-blocking, and
+        // never throws out of this method — diagnostics must never themselves crash.
+        private static void appendToBoth(string payload)
+        {
+            tryAppend(internalDir, payload);
+            tryAppend(externalDir, payload);
+        }
 
-                string path = Path.Combine(internalDir, CRASH_LOG_NAME);
+        private static void tryAppend(string? dir, string payload)
+        {
+            if (dir == null) return;
+
+            try
+            {
+                string path = Path.Combine(dir, CRASH_LOG_NAME);
                 using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite);
                 using var sw = new StreamWriter(fs);
-
-                sw.WriteLine();
-                sw.WriteLine("=========================================================");
-                sw.WriteLine("=== MANAGED UNHANDLED EXCEPTION ===");
-                sw.WriteLine($"  source     = {source}");
-                sw.WriteLine($"  utc_time   = {DateTime.UtcNow:O}");
-                sw.WriteLine($"  thread_id  = {Environment.CurrentManagedThreadId}");
-                sw.WriteLine();
-                sw.WriteLine(ex?.ToString() ?? "<no exception object>");
-                sw.WriteLine("=== END OF MANAGED EXCEPTION ===");
-                sw.WriteLine();
+                sw.Write(payload);
                 sw.Flush();
-                fs.Flush(true);
             }
             catch (Exception e)
             {
-                Debug.WriteLine($"[osu!] writeManagedException failed: {e.Message}");
+                Debug.WriteLine($"[osu!] CrashDiagnostics.tryAppend({dir}) failed: {e.Message}");
             }
         }
 

osu.Android/Native/OboeAudioBridge.cs

@@ -216,5 +216,6 @@ public void Dispose()
         [DllImport(lib_name)] internal static extern void nADPFUpdateTargetDuration(IntPtr sessionPtr, long targetDurationNanos);
         [DllImport(lib_name)] internal static extern void nADPFCloseSession(IntPtr sessionPtr);
         [DllImport(lib_name)] internal static extern void nInstallCrashHandler([MarshalAs(UnmanagedType.LPUTF8Str)] string? logPath);
+        [DllImport(lib_name)] internal static extern void nReinstallCrashHandler();
     }
 }

osu.Android/Native/crash_handler.cpp

@@ -1006,4 +1006,40 @@ void nInstallCrashHandler(const char* logPath) {
         g_logPath[0] ? g_logPath : "<none, logcat only>");
 }
 
+// Re-install the signal handlers without short-circuiting on g_installed.
+// Mono installs its own SIGSEGV handler later in startup (after activity
+// OnCreate), which sits in front of ours and intercepts JIT null-deref
+// faults — re-raising via tgkill when it cannot translate them, which
+// bypasses our dump.  Calling this from a later phase (e.g. GameHost.Run)
+// puts our handler back on top of the chain, with Mono's saved as the
+// previous handler so chaining still works.
+__attribute__((visibility("default")))
+void nReinstallCrashHandler() {
+    // Re-install alt stack (cheap; idempotent on the same buffer).
+    stack_t ss{};
+    ss.ss_sp = g_altStack;
+    ss.ss_size = kAltStackSize;
+    ss.ss_flags = 0;
+    sigaltstack(&ss, nullptr);
+
+    struct sigaction sa{};
+    sa.sa_sigaction = &crashHandler;
+    sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_RESTART;
+    sigemptyset(&sa.sa_mask);
+
+    for (size_t i = 0; i < kNumSignals; ++i) {
+        // Overwrite previous-handler slot with whatever is currently
+        // installed (typically Mono's handler at this point), so when our
+        // handler chains, it forwards to Mono rather than to our own
+        // already-saved entry.
+        sigaction(kSignals[i], &sa, &g_prevHandlers[i]);
+    }
+
+    g_installed = 1;
+
+    __android_log_print(ANDROID_LOG_INFO, CRASH_LOG_TAG,
+        "Crash handler re-installed (logPath=%s)",
+        g_logPath[0] ? g_logPath : "<none, logcat only>");
+}
+
 } // extern "C"

osu.Android/Native/crash_handler.h

@@ -9,4 +9,13 @@ extern "C" {
 // external files directory).  Pass nullptr to log only to logcat.
 // Idempotent: subsequent calls after the first are no-ops.
 void nInstallCrashHandler(const char* logPath);
+
+// Re-install the signal handlers, even if a previous install already ran.
+// Use this after the Mono runtime is fully up so our handler chains *on top
+// of* Mono's SIGSEGV handler — otherwise Mono intercepts JIT-NRE faults
+// first and re-raises via `tgkill` (which appears in tombstones as
+// `si_code = SI_TKILL`), bypassing our dump entirely.  The previously-saved
+// "previous handler" slot is overwritten with whatever is currently
+// installed (typically Mono's), so chaining still works.
+void nReinstallCrashHandler();
 }

osu.Android/OsuGameActivity.cs

@@ -94,6 +94,7 @@ protected override void OnCreate(Bundle? savedInstanceState)
             CrashDiagnostics.InstallNativeHandler(this);
             CrashDiagnostics.InstallManagedExceptionHooks();
             CrashDiagnostics.WriteAliveMarker("Activity.OnCreate entry");
+            CrashDiagnostics.WriteInstallState();
             CrashDiagnostics.MirrorInternalLogToExternal();
 
             base.OnCreate(savedInstanceState);

osu.Android/OsuGameAndroid.cs

@@ -762,8 +762,16 @@ public override void SetHost(GameHost host)
         {
             CrashDiagnostics.WriteAliveMarker("OsuGameAndroid.SetHost (GameHost.Run entry)");
 
+            // Re-install the native crash handler now that the Mono runtime has had a chance
+            // to install its own SIGSEGV handler. Without this, Mono sits in front of us in
+            // the chain and intercepts JIT null-deref faults — re-raising via tgkill (visible
+            // as si_code = SI_TKILL in tombstones) without forwarding to our dump.
+            CrashDiagnostics.ReinstallNativeHandler();
+
             base.SetHost(host);
 
+            CrashDiagnostics.WriteAliveMarker("OsuGameAndroid.SetHost (base.SetHost returned)");
+
             if (host.Window != null)
                 host.Window.CursorState |= CursorState.Hidden;
         }