Merge pull request #250 from winnerspiros/copilot/fix-black-screen-issue-yet-again

Android: pthread liveness watchdog + first-load mitigations for startup hang

Author: winnerspiros

osu.Android/AndroidStartupFlags.cs

@@ -0,0 +1,108 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.IO;
+using Android.App;
+using Debug = System.Diagnostics.Debug;
+
+namespace osu.Android
+{
+    /// <summary>
+    /// Tiny on-disk sentinel store that lets pre-<see cref="osu.Game.OsuGameBase"/> code
+    /// (e.g. <see cref="OsuGameActivity.OnCreate"/>) consult settings owned by
+    /// <see cref="osu.Game.Configuration.OsuConfigManager"/>.
+    ///
+    /// <para>
+    /// The activity runs LONG before the config manager exists — Realm has to be
+    /// initialised first, and the activity is the place that needs to RUN safety
+    /// behaviours BEFORE Realm runs. The chicken-and-egg solution is a one-flag-
+    /// per-file sentinel pattern: when the user changes a startup-safety toggle
+    /// in-game, <see cref="OsuGameAndroid"/> writes (or deletes) a sentinel file
+    /// whose presence reflects the new value. The activity reads the sentinel
+    /// next launch.
+    /// </para>
+    ///
+    /// <para>
+    /// Files live under <c>FilesDir</c> (internal app storage). Each flag is a
+    /// single empty file named <c>android_startup_disable_&lt;name&gt;.flag</c>.
+    /// Presence ⇒ "the user has explicitly disabled this safety net". Absence ⇒
+    /// "default behaviour" (per the matching OsuSetting default in
+    /// <see cref="osu.Game.Configuration.OsuConfigManager"/>).
+    /// </para>
+    ///
+    /// <para>
+    /// All operations are best-effort and never throw out — diagnostics-grade
+    /// reliability semantics, like <see cref="CrashDiagnostics"/>.
+    /// </para>
+    /// </summary>
+    internal static class AndroidStartupFlags
+    {
+        public const string FLAG_CLEANUP_REALM_FIFOS_DISABLED = "android_startup_disable_realm_fifo_cleanup.flag";
+        public const string FLAG_DEFER_NATIVE_INIT_DISABLED = "android_startup_disable_defer_native_init.flag";
+        public const string FLAG_FRAME_SYNC_MIGRATION_ENABLED = "android_startup_enable_frame_sync_migration.flag";
+
+        private static string? resolveDir()
+        {
+            try
+            {
+                var ctx = Application.Context;
+                var files = ctx?.FilesDir;
+                if (files == null) return null;
+                string? path = files.AbsolutePath;
+                return string.IsNullOrEmpty(path) ? null : path;
+            }
+            catch (Exception e)
+            {
+                Debug.WriteLine($"[osu!] AndroidStartupFlags.resolveDir failed: {e.Message}");
+                return null;
+            }
+        }
+
+        /// <summary>
+        /// Returns true if a sentinel file with the given name exists in internal app storage.
+        /// </summary>
+        public static bool IsSet(string flagName)
+        {
+            try
+            {
+                string? dir = resolveDir();
+                if (dir == null) return false;
+                return File.Exists(Path.Combine(dir, flagName));
+            }
+            catch (Exception e)
+            {
+                Debug.WriteLine($"[osu!] AndroidStartupFlags.IsSet({flagName}) failed: {e.Message}");
+                return false;
+            }
+        }
+
+        /// <summary>
+        /// Create or remove the sentinel file according to <paramref name="set"/>.
+        /// Idempotent and never throws.
+        /// </summary>
+        public static void Set(string flagName, bool set)
+        {
+            try
+            {
+                string? dir = resolveDir();
+                if (dir == null) return;
+                string path = Path.Combine(dir, flagName);
+                if (set)
+                {
+                    if (!File.Exists(path))
+                        File.WriteAllText(path, string.Empty);
+                }
+                else
+                {
+                    if (File.Exists(path))
+                        File.Delete(path);
+                }
+            }
+            catch (Exception e)
+            {
+                Debug.WriteLine($"[osu!] AndroidStartupFlags.Set({flagName}, {set}) failed: {e.Message}");
+            }
+        }
+    }
+}

osu.Android/CrashDiagnostics.cs

@@ -150,6 +150,40 @@ public static void ReinstallNativeHandler()
             }
         }
 
+        /// <summary>
+        /// Arm the native pthread liveness watchdog. Writes its hang dumps to the same
+        /// internal-storage <c>native_crash.log</c> the rest of the diagnostics pipeline uses.
+        ///
+        /// <para>
+        /// The native watchdog is the only diagnostic that survives a Mono stop-the-world GC
+        /// pause: it runs as a pthread that never attaches to the runtime, so Mono cannot
+        /// suspend it during STW. This is essential for diagnosing the "every managed thread
+        /// parked in <c>__rt_sigsuspend</c>" startup hangs we have been chasing — under that
+        /// failure mode the managed <see cref="osu.Android.HangWatchdog"/> is itself frozen
+        /// and produces no dump.
+        /// </para>
+        ///
+        /// <para>
+        /// Idempotent on the native side; safe to call from any thread; never throws.
+        /// Caller is expected to gate on <c>OsuSetting.AndroidNativeWatchdogEnabled</c> so
+        /// the user can disable the diagnostic from in-game settings if it ever interferes
+        /// with normal operation.
+        /// </para>
+        /// </summary>
+        /// <param name="hangSeconds">Threshold (clamped to [3, 120] on the native side).</param>
+        public static void StartNativeWatchdog(int hangSeconds)
+        {
+            try
+            {
+                NativeWatchdog.Start(installedLogPath, hangSeconds);
+                WriteAliveMarker($"CrashDiagnostics.StartNativeWatchdog (threshold={hangSeconds}s)");
+            }
+            catch (Exception e)
+            {
+                Debug.WriteLine($"[osu!] StartNativeWatchdog failed: {e.Message}");
+            }
+        }
+
         /// <summary>
         /// Append a single-line "I am alive" marker to the crash log so that, when we later
         /// inspect a truncated/empty file after a crash, the last-written marker pinpoints

osu.Android/HangWatchdog.cs

@@ -401,6 +401,15 @@ private void tick()
             {
                 Interlocked.Exchange(ref LastTickUtcMs, nowUtcMs());
 
+                // Bump the native pthread watchdog. Cheap (one __atomic_store_n
+                // on a 64-bit slot) and only does anything if the libosu_native.so
+                // entry point is present; otherwise the wrapper silently no-ops.
+                // The native watchdog is what produces hang dumps when Mono's
+                // STW GC has frozen every managed thread — without this
+                // heartbeat its timer-driven loop has nothing to compare against
+                // and would dump on first tick.
+                osu.Android.Native.NativeWatchdog.Heartbeat();
+
                 // gettid is cheap (single syscall) and only meaningfully
                 // changes on the very first tick — but we re-record it on
                 // every tick so a thread restart (e.g. ExecutionMode swap)

osu.Android/LogManagement.cs

@@ -25,15 +25,14 @@ namespace osu.Android
     /// </para>
     ///
     /// <para>
-    /// We additionally lower <see cref="Logger.Level"/> to
-    /// <see cref="LogLevel.Important"/> on Android. The default for release
-    /// builds is <see cref="LogLevel.Verbose"/>, which on osu! emits one
-    /// <c>[verbose]</c> line per shader compile, per texture upload, per SDL
-    /// no-op, per `Loading X...` event, etc. Almost none of that helps debug a
-    /// startup ANR — the actionable signal is exclusively in <c>[important]</c>
-    /// and <c>[error]</c> entries (Vulkan surface-lost events, mode changes,
-    /// realm migration, exceptions). Cutting verbose drops the typical per-launch
-    /// log size by ~95% without losing any diagnostic value.
+    /// Historically this class also lowered <see cref="Logger.Level"/> to
+    /// <see cref="LogLevel.Important"/> on Android. That has been reverted —
+    /// the framework's default verbosity is restored so osu.log captures the
+    /// full per-thread startup narrative needed to diagnose hangs. On-disk
+    /// log size is still bounded by <see cref="pruneLogDirectory"/> at every
+    /// startup (oldest-first eviction down to <see cref="MAX_LOG_BYTES"/>),
+    /// so verbose logs cannot regress the ~480 MB footprint that originally
+    /// motivated this file.
     /// </para>
     /// </summary>
     internal static class LogManagement
@@ -59,18 +58,15 @@ internal static class LogManagement
         /// </summary>
         public static void Apply()
         {
-            try
-            {
-                // Setting Logger.Level is a no-op on entries already queued, but
-                // since we run from Activity.OnCreate before the framework host
-                // has been constructed (and therefore before the first framework
-                // log entry), this takes effect for the entire session.
-                Logger.Level = LogLevel.Important;
-            }
-            catch (Exception e)
-            {
-                Debug.WriteLine($"[osu!] LogManagement: could not lower Logger.Level: {e.Message}");
-            }
+            // NOTE: we used to force Logger.Level = LogLevel.Important here to
+            // shrink runtime log output during the "log explosion" debugging
+            // window. That has been reverted at user request — the default
+            // framework log verbosity is now restored so osu.log captures the
+            // full per-thread startup narrative we need to diagnose hangs.
+            // Log size is still bounded by pruneLogDirectory() below
+            // (MAX_LOG_BYTES cap with oldest-first eviction), so re-enabling
+            // verbose logging cannot regress the on-disk footprint that the
+            // 480 MB report originally exposed.
 
             try
             {

osu.Android/Native/CMakeLists.txt

@@ -86,6 +86,7 @@ add_library(osu_native SHARED
     oboe_bridge.cpp
     vulkan_bridge.cpp
     crash_handler.cpp
+    native_watchdog.cpp
 )
 
 

osu.Android/Native/NativeWatchdog.cs

@@ -0,0 +1,96 @@
+// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
+// See the LICENCE file in the repository root for full licence text.
+
+using System;
+using System.Runtime.InteropServices;
+using Debug = System.Diagnostics.Debug;
+
+namespace osu.Android.Native
+{
+    /// <summary>
+    /// Managed-side wrapper for the native pthread liveness watchdog implemented in
+    /// <c>osu.Android/Native/native_watchdog.cpp</c>.
+    ///
+    /// <para>
+    /// The native watchdog exists because the managed <see cref="osu.Android.HangWatchdog"/>
+    /// runs as a normal <c>System.Threading.Thread</c>, which Mono suspends during a
+    /// stop-the-world GC by sending <c>SIGRTMIN+N</c>. If a Mono thread is stuck
+    /// inside a long native call (Vulkan present-queue futex, Realm fifo open,
+    /// AAudio init, …) the STW request never completes and every other managed
+    /// thread — including our managed watchdog's monitor — is parked indefinitely.
+    /// A pure pthread-only watchdog that never attaches to Mono is the only thing
+    /// that can produce a diagnostic dump under that condition.
+    /// </para>
+    ///
+    /// <para>
+    /// All entry points are best-effort: a missing native library or a
+    /// <see cref="DllNotFoundException"/> is non-fatal and silently downgraded to
+    /// a <see cref="Debug.WriteLine"/> call so startup is unaffected.
+    /// </para>
+    /// </summary>
+    internal static class NativeWatchdog
+    {
+        // Same lib name used by OboeAudioBridge — single shared libosu_native.so.
+        private const string lib_name = "osu_native";
+
+        /// <summary>
+        /// Arm the native watchdog with the given log path and hang threshold.
+        /// Idempotent: subsequent calls are no-ops on the native side.
+        /// Never throws.
+        /// </summary>
+        /// <param name="logPath">Absolute path of the file to append hang dumps to (typically <c>FilesDir/native_crash.log</c>).</param>
+        /// <param name="hangSeconds">Seconds without a heartbeat before a dump is triggered. Native side clamps to [3, 120].</param>
+        public static void Start(string? logPath, int hangSeconds)
+        {
+            try
+            {
+                osu_native_watchdog_start(logPath, hangSeconds);
+            }
+            catch (DllNotFoundException e)
+            {
+                Debug.WriteLine($"[osu!] NativeWatchdog.Start: libosu_native.so not loaded, watchdog disabled ({e.Message})");
+            }
+            catch (EntryPointNotFoundException e)
+            {
+                // The native entry point is absent — most likely an old libosu_native.so
+                // in the APK that does not include native_watchdog.cpp. Treat as disabled
+                // rather than crashing the user's startup.
+                Debug.WriteLine($"[osu!] NativeWatchdog.Start: entry point missing, watchdog disabled ({e.Message})");
+            }
+            catch (Exception e)
+            {
+                Debug.WriteLine($"[osu!] NativeWatchdog.Start unexpected failure: {e.Message}");
+            }
+        }
+
+        /// <summary>
+        /// Bump the native heartbeat. Called from the managed
+        /// <see cref="osu.Android.HangWatchdog"/> per-thread tick so the native watchdog
+        /// can observe Update-thread liveness across Mono STW pauses. The underlying
+        /// native call performs a single <c>__atomic_store_n</c> on a 64-bit slot;
+        /// safe to call at any rate, from any thread, without locking.
+        /// Never throws.
+        /// </summary>
+        public static void Heartbeat()
+        {
+            try
+            {
+                osu_native_watchdog_heartbeat();
+            }
+            catch (DllNotFoundException) { /* watchdog disabled — no-op */ }
+            catch (EntryPointNotFoundException) { /* old libosu_native.so — no-op */ }
+            catch (Exception e)
+            {
+                // Heartbeat is on the GameThread tick path; never let a diagnostic
+                // failure escape into the game loop.
+                Debug.WriteLine($"[osu!] NativeWatchdog.Heartbeat unexpected failure: {e.Message}");
+            }
+        }
+
+        [DllImport(lib_name)]
+        private static extern void osu_native_watchdog_start([MarshalAs(UnmanagedType.LPUTF8Str)] string? logPath, int hangSeconds);
+
+        [DllImport(lib_name)]
+        private static extern void osu_native_watchdog_heartbeat();
+    }
+}