forked from ppy/osu
-
Notifications
You must be signed in to change notification settings - Fork 0
Android: capture JIT-thread SIGSEGV by chaining handler on top of Mono #238
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Merged
Merged
Changes from all commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||
|---|---|---|---|---|---|---|---|---|
|
|
@@ -15,13 +15,14 @@ namespace osu.Android | |||||||
| /// <summary> | ||||||||
| /// Centralised Android crash-diagnostics plumbing. | ||||||||
| /// | ||||||||
| /// We write everything to <b>internal</b> app storage (<c>FilesDir</c>) because external | ||||||||
| /// storage is FUSE-backed, scoped-storage-restricted, and may not be ready at the | ||||||||
| /// instant a very-early crash hits. On the next normal startup we mirror the internal | ||||||||
| /// crash log to <c>GetExternalFilesDir(null)</c> so the user can grab it via the Files | ||||||||
| /// app on an unrooted device, then truncate the internal copy. | ||||||||
| /// We write everything to <b>both</b> internal app storage (<c>FilesDir</c>) and external | ||||||||
| /// app storage (<c>GetExternalFilesDir(null)</c>) when both are available. Internal is the | ||||||||
| /// reliable target for the very-early window where external storage may not yet be ready; | ||||||||
| /// external is reachable by the user via the Files app on an unrooted device and receives | ||||||||
| /// alive markers / managed-exception dumps in real time so the user does not have to wait | ||||||||
| /// for a successful next startup to mirror the data over. | ||||||||
| /// | ||||||||
| /// Files (all relative to <c>FilesDir</c>): | ||||||||
| /// Files (relative to each storage dir): | ||||||||
| /// <list type="bullet"> | ||||||||
| /// <item><c>native_crash.log</c> — append target for both the native handler and the managed last-chance hooks; also receives "I am alive" startup markers.</item> | ||||||||
| /// <item><c>crash_handler_installed.txt</c> — sentinel dropped immediately after <c>nInstallCrashHandler</c> returns. Lets us distinguish "handler never installed (P/Invoke failed → libosu_native.so missing)" from "handler installed but signal bypassed it".</item> | ||||||||
|
|
@@ -37,6 +38,9 @@ internal static class CrashDiagnostics | |||||||
|
|
||||||||
| private static string? internalDir; | ||||||||
| private static string? externalDir; | ||||||||
| private static string? sentinelPath; | ||||||||
| private static string? installedLogPath; | ||||||||
| private static bool sentinelWritten; | ||||||||
|
|
||||||||
| /// <summary> | ||||||||
| /// Installs the native crash handler against the internal-storage log path, drops the | ||||||||
|
|
@@ -51,22 +55,24 @@ public static void InstallNativeHandler(Context context) | |||||||
| { | ||||||||
| resolveDirs(context); | ||||||||
|
|
||||||||
| string? logPath = internalDir != null ? Path.Combine(internalDir, CRASH_LOG_NAME) : null; | ||||||||
| installedLogPath = internalDir != null ? Path.Combine(internalDir, CRASH_LOG_NAME) : null; | ||||||||
|
|
||||||||
| // The native handler is best-effort. Wrap so a DllNotFoundException | ||||||||
| // (libosu_native.so missing from the APK) cannot itself crash us. | ||||||||
| try | ||||||||
| { | ||||||||
| OboeAudioBridge.nInstallCrashHandler(logPath); | ||||||||
| OboeAudioBridge.nInstallCrashHandler(installedLogPath); | ||||||||
|
|
||||||||
| // Sentinel: only written when nInstallCrashHandler returned without throwing. | ||||||||
| if (internalDir != null) | ||||||||
| { | ||||||||
| try | ||||||||
| { | ||||||||
| sentinelPath = Path.Combine(internalDir, SENTINEL_NAME); | ||||||||
| File.WriteAllText( | ||||||||
| Path.Combine(internalDir, SENTINEL_NAME), | ||||||||
| $"installed_at={DateTime.UtcNow:O}\nlog_path={logPath ?? "<none>"}\n"); | ||||||||
| sentinelPath, | ||||||||
| $"installed_at={DateTime.UtcNow:O}\nlog_path={installedLogPath ?? "<none>"}\n"); | ||||||||
| sentinelWritten = true; | ||||||||
| } | ||||||||
| catch (Exception e) { Debug.WriteLine($"[osu!] Could not write crash-handler sentinel: {e.Message}"); } | ||||||||
| } | ||||||||
|
|
@@ -87,31 +93,38 @@ public static void InstallNativeHandler(Context context) | |||||||
| } | ||||||||
|
|
||||||||
| /// <summary> | ||||||||
| /// Append a single-line "I am alive" marker to the internal crash log so that, when | ||||||||
| /// we later inspect a truncated/empty file after a crash, the last-written marker | ||||||||
| /// pinpoints which startup phase died. | ||||||||
| /// Re-install the native signal handlers from a later startup phase, after the Mono | ||||||||
| /// runtime has installed its own SIGSEGV handler. This is what actually lets us catch | ||||||||
| /// JIT-thread null-deref crashes — without it, Mono's handler intercepts the fault | ||||||||
| /// first and re-raises via <c>tgkill</c> (visible in tombstones as | ||||||||
| /// <c>si_code = SI_TKILL</c>) without ever forwarding to us. | ||||||||
| /// </summary> | ||||||||
| public static void WriteAliveMarker(string phase) | ||||||||
| public static void ReinstallNativeHandler() | ||||||||
| { | ||||||||
| try | ||||||||
| { | ||||||||
| if (internalDir == null) return; | ||||||||
|
|
||||||||
| string path = Path.Combine(internalDir, CRASH_LOG_NAME); | ||||||||
| string line = $"=== ALIVE [{DateTime.UtcNow:O}] {phase} ===\n"; | ||||||||
|
|
||||||||
| // Append using a bounded write — never throw, never block. | ||||||||
| using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite); | ||||||||
| using var sw = new StreamWriter(fs); | ||||||||
| sw.Write(line); | ||||||||
| sw.Flush(); | ||||||||
| OboeAudioBridge.nReinstallCrashHandler(); | ||||||||
| WriteAliveMarker("CrashDiagnostics.ReinstallNativeHandler (chained on top of Mono)"); | ||||||||
| } | ||||||||
| catch (Exception e) | ||||||||
| { | ||||||||
| Debug.WriteLine($"[osu!] WriteAliveMarker({phase}) failed: {e.Message}"); | ||||||||
| Debug.WriteLine($"[osu!] nReinstallCrashHandler P/Invoke failed: {e.Message}"); | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| /// <summary> | ||||||||
| /// Append a single-line "I am alive" marker to the crash log so that, when we later | ||||||||
| /// inspect a truncated/empty file after a crash, the last-written marker pinpoints | ||||||||
| /// which startup phase died. Writes to both internal and external storage so the user | ||||||||
| /// can pull the file immediately without waiting for a successful next startup to | ||||||||
| /// mirror it over. | ||||||||
| /// </summary> | ||||||||
| public static void WriteAliveMarker(string phase) | ||||||||
| { | ||||||||
| string line = $"=== ALIVE [{DateTime.UtcNow:O}] {phase} ===\n"; | ||||||||
| appendToBoth(line); | ||||||||
| } | ||||||||
|
|
||||||||
| /// <summary> | ||||||||
| /// One-shot post-crash mirror: if an internal <c>native_crash.log</c> exists and is | ||||||||
| /// non-empty, copy it to external app storage (so the user can pull it via the Files | ||||||||
|
|
@@ -179,34 +192,114 @@ public static void InstallManagedExceptionHooks() | |||||||
| writeManagedException("TaskScheduler.UnobservedTaskException", e.Exception); | ||||||||
| // Don't mark observed — the framework / sentry pipeline still wants to see it. | ||||||||
| }; | ||||||||
|
|
||||||||
| // FirstChanceException fires for *every* managed exception, even ones that get | ||||||||
| // caught later. On non-main managed threads (e.g. the Draw thread), Mono on | ||||||||
| // Android does not always route an unhandled exception through | ||||||||
| // AppDomain.UnhandledException before aborting — so without this hook the | ||||||||
| // exception that ultimately kills the process can vanish without trace. We | ||||||||
| // record it here on every throw so the *last* recorded exception before a | ||||||||
| // SIGSEGV/SIGABRT is the candidate culprit. To avoid drowning the log in noise | ||||||||
| // we filter by exception type — only fatal-ish kinds are recorded. | ||||||||
| try | ||||||||
| { | ||||||||
| AppDomain.CurrentDomain.FirstChanceException += (_, e) => | ||||||||
| { | ||||||||
| if (e.Exception is NullReferenceException | ||||||||
| or AccessViolationException | ||||||||
| or StackOverflowException | ||||||||
| or TypeInitializationException | ||||||||
| or DllNotFoundException | ||||||||
| or EntryPointNotFoundException | ||||||||
| or BadImageFormatException | ||||||||
| or TypeLoadException | ||||||||
| or MissingMethodException | ||||||||
| or MissingFieldException | ||||||||
| or InvalidProgramException) | ||||||||
| { | ||||||||
| writeManagedException($"FirstChanceException ({e.Exception.GetType().Name})", e.Exception); | ||||||||
| } | ||||||||
| }; | ||||||||
| } | ||||||||
| catch (Exception e) | ||||||||
| { | ||||||||
| Debug.WriteLine($"[osu!] Could not install FirstChanceException hook: {e.Message}"); | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| /// <summary> | ||||||||
| /// Records a one-line summary of the native handler install state (sentinel exists? | ||||||||
| /// log path?) so the very first thing we see in the log on the next inspection tells | ||||||||
| /// us whether the native handler is even in place. | ||||||||
| /// </summary> | ||||||||
| public static void WriteInstallState() | ||||||||
| { | ||||||||
| try | ||||||||
| { | ||||||||
| string sentinelState; | ||||||||
|
|
||||||||
| if (sentinelWritten && sentinelPath != null && File.Exists(sentinelPath)) | ||||||||
| sentinelState = "present"; | ||||||||
| else if (sentinelWritten) | ||||||||
| sentinelState = "written-but-missing"; | ||||||||
| else | ||||||||
| sentinelState = "absent"; | ||||||||
|
|
||||||||
| appendToBoth($"=== INSTALL_STATE sentinel={sentinelState} log_path={installedLogPath ?? "<none>"} internal_dir={internalDir ?? "<none>"} external_dir={externalDir ?? "<none>"} ===\n"); | ||||||||
| } | ||||||||
| catch (Exception e) | ||||||||
| { | ||||||||
| Debug.WriteLine($"[osu!] WriteInstallState failed: {e.Message}"); | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| private static void writeManagedException(string source, Exception? ex) | ||||||||
| { | ||||||||
| try | ||||||||
| { | ||||||||
| if (internalDir == null) return; | ||||||||
| string block = | ||||||||
| "\n=========================================================\n" + | ||||||||
| "=== MANAGED EXCEPTION ===\n" + | ||||||||
| $" source = {source}\n" + | ||||||||
| $" utc_time = {DateTime.UtcNow:O}\n" + | ||||||||
| $" thread_id = {Environment.CurrentManagedThreadId}\n" + | ||||||||
| $" thread_name= {Thread.CurrentThread.Name ?? "<null>"}\n" + | ||||||||
| "\n" + | ||||||||
| (ex?.ToString() ?? "<no exception object>") + "\n" + | ||||||||
| "=== END OF MANAGED EXCEPTION ===\n\n"; | ||||||||
| appendToBoth(block); | ||||||||
| } | ||||||||
| catch (Exception e) | ||||||||
| { | ||||||||
| Debug.WriteLine($"[osu!] writeManagedException failed: {e.Message}"); | ||||||||
| } | ||||||||
| } | ||||||||
|
|
||||||||
| // Append the same payload to both internal (FilesDir) and external (GetExternalFilesDir) | ||||||||
| // crash logs. Either may legitimately be unavailable; failure of one path must not | ||||||||
| // prevent the other from being written. Each write is bounded, non-blocking, and | ||||||||
| // never throws out of this method — diagnostics must never themselves crash. | ||||||||
| private static void appendToBoth(string payload) | ||||||||
| { | ||||||||
| tryAppend(internalDir, payload); | ||||||||
| tryAppend(externalDir, payload); | ||||||||
| } | ||||||||
|
|
||||||||
| string path = Path.Combine(internalDir, CRASH_LOG_NAME); | ||||||||
| private static void tryAppend(string? dir, string payload) | ||||||||
| { | ||||||||
| if (dir == null) return; | ||||||||
|
|
||||||||
| try | ||||||||
| { | ||||||||
| string path = Path.Combine(dir, CRASH_LOG_NAME); | ||||||||
| using var fs = new FileStream(path, FileMode.Append, FileAccess.Write, FileShare.ReadWrite); | ||||||||
| using var sw = new StreamWriter(fs); | ||||||||
|
|
||||||||
| sw.WriteLine(); | ||||||||
| sw.WriteLine("========================================================="); | ||||||||
| sw.WriteLine("=== MANAGED UNHANDLED EXCEPTION ==="); | ||||||||
| sw.WriteLine($" source = {source}"); | ||||||||
| sw.WriteLine($" utc_time = {DateTime.UtcNow:O}"); | ||||||||
| sw.WriteLine($" thread_id = {Environment.CurrentManagedThreadId}"); | ||||||||
| sw.WriteLine(); | ||||||||
| sw.WriteLine(ex?.ToString() ?? "<no exception object>"); | ||||||||
| sw.WriteLine("=== END OF MANAGED EXCEPTION ==="); | ||||||||
| sw.WriteLine(); | ||||||||
| sw.Write(payload); | ||||||||
| sw.Flush(); | ||||||||
|
||||||||
| sw.Flush(); | |
| sw.Flush(); | |
| fs.Flush(true); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||||||||||||||||||||||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -1006,4 +1006,40 @@ void nInstallCrashHandler(const char* logPath) { | |||||||||||||||||||||||||||||||||||||
| g_logPath[0] ? g_logPath : "<none, logcat only>"); | ||||||||||||||||||||||||||||||||||||||
| } | ||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||
| // Re-install the signal handlers without short-circuiting on g_installed. | ||||||||||||||||||||||||||||||||||||||
| // Mono installs its own SIGSEGV handler later in startup (after activity | ||||||||||||||||||||||||||||||||||||||
| // OnCreate), which sits in front of ours and intercepts JIT null-deref | ||||||||||||||||||||||||||||||||||||||
| // faults — re-raising via tgkill when it cannot translate them, which | ||||||||||||||||||||||||||||||||||||||
| // bypasses our dump. Calling this from a later phase (e.g. GameHost.Run) | ||||||||||||||||||||||||||||||||||||||
| // puts our handler back on top of the chain, with Mono's saved as the | ||||||||||||||||||||||||||||||||||||||
| // previous handler so chaining still works. | ||||||||||||||||||||||||||||||||||||||
| __attribute__((visibility("default"))) | ||||||||||||||||||||||||||||||||||||||
| void nReinstallCrashHandler() { | ||||||||||||||||||||||||||||||||||||||
| // Re-install alt stack (cheap; idempotent on the same buffer). | ||||||||||||||||||||||||||||||||||||||
| stack_t ss{}; | ||||||||||||||||||||||||||||||||||||||
| ss.ss_sp = g_altStack; | ||||||||||||||||||||||||||||||||||||||
| ss.ss_size = kAltStackSize; | ||||||||||||||||||||||||||||||||||||||
| ss.ss_flags = 0; | ||||||||||||||||||||||||||||||||||||||
| sigaltstack(&ss, nullptr); | ||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||
| struct sigaction sa{}; | ||||||||||||||||||||||||||||||||||||||
| sa.sa_sigaction = &crashHandler; | ||||||||||||||||||||||||||||||||||||||
| sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_RESTART; | ||||||||||||||||||||||||||||||||||||||
| sigemptyset(&sa.sa_mask); | ||||||||||||||||||||||||||||||||||||||
|
|
||||||||||||||||||||||||||||||||||||||
| for (size_t i = 0; i < kNumSignals; ++i) { | ||||||||||||||||||||||||||||||||||||||
| // Overwrite previous-handler slot with whatever is currently | ||||||||||||||||||||||||||||||||||||||
| // installed (typically Mono's handler at this point), so when our | ||||||||||||||||||||||||||||||||||||||
| // handler chains, it forwards to Mono rather than to our own | ||||||||||||||||||||||||||||||||||||||
| // already-saved entry. | ||||||||||||||||||||||||||||||||||||||
| sigaction(kSignals[i], &sa, &g_prevHandlers[i]); | ||||||||||||||||||||||||||||||||||||||
|
Comment on lines
+1031
to
+1035
|
||||||||||||||||||||||||||||||||||||||
| // Overwrite previous-handler slot with whatever is currently | |
| // installed (typically Mono's handler at this point), so when our | |
| // handler chains, it forwards to Mono rather than to our own | |
| // already-saved entry. | |
| sigaction(kSignals[i], &sa, &g_prevHandlers[i]); | |
| struct sigaction oldact{}; | |
| // Save the currently installed handler so we can keep chaining to it, | |
| // but never replace our saved previous handler with ourselves. | |
| if (sigaction(kSignals[i], &sa, &oldact) == 0) { | |
| const bool oldIsCrashHandler = | |
| ((oldact.sa_flags & SA_SIGINFO) != 0) | |
| ? (oldact.sa_sigaction == &crashHandler) | |
| : (oldact.sa_handler == reinterpret_cast<void (*)(int)>(&crashHandler)); | |
| if (!oldIsCrashHandler) | |
| g_prevHandlers[i] = oldact; | |
| } |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||||||||||
|---|---|---|---|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -94,6 +94,7 @@ protected override void OnCreate(Bundle? savedInstanceState) | |||||||||||||
| CrashDiagnostics.InstallNativeHandler(this); | ||||||||||||||
| CrashDiagnostics.InstallManagedExceptionHooks(); | ||||||||||||||
| CrashDiagnostics.WriteAliveMarker("Activity.OnCreate entry"); | ||||||||||||||
| CrashDiagnostics.WriteInstallState(); | ||||||||||||||
| CrashDiagnostics.MirrorInternalLogToExternal(); | ||||||||||||||
|
Comment on lines
96
to
98
|
||||||||||||||
| CrashDiagnostics.WriteAliveMarker("Activity.OnCreate entry"); | |
| CrashDiagnostics.WriteInstallState(); | |
| CrashDiagnostics.MirrorInternalLogToExternal(); | |
| CrashDiagnostics.MirrorInternalLogToExternal(); | |
| CrashDiagnostics.WriteAliveMarker("Activity.OnCreate entry"); | |
| CrashDiagnostics.WriteInstallState(); |
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Now that appendToBoth() writes markers/exceptions to external storage in real time, the existing MirrorInternalLogToExternal() call on startup is likely to append duplicate content into the external log (and then truncate internal). Consider adding a guard/flag so mirroring only occurs when external logging was unavailable during the previous run, or otherwise ensure the mirror can't duplicate content produced by appendToBoth().