Skip to content

Android Input Latency and S Pen Optimizations #148

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
winnerspiros merged 4 commits into from
Apr 4, 2026
Merged

Android Input Latency and S Pen Optimizations #148

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
4fa6a23
android: optimize input latency and fix S Pen recognition
google-labs-jules[bot] Apr 4, 2026
81461dd
android: fix build errors and optimize input latency
google-labs-jules[bot] Apr 4, 2026
d232664
android: fix input handler build errors and refine latency optimizations
google-labs-jules[bot] Apr 4, 2026
7789155
android: optimize input latency and preserve platform members
google-labs-jules[bot] Apr 4, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 133 additions & 0 deletions osu.Android/Input/AndroidKeyboardHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.

using Android.Views;
using osu.Framework.Input.Handlers;
using osu.Framework.Input.StateChanges;
using osu.Framework.Platform;
using osuTK.Input;

namespace osu.Android.Input
{
public class AndroidKeyboardHandler : InputHandler
{
public override string Description => "Keyboard (Low Latency)";
public override bool IsActive => Enabled.Value;

public AndroidKeyboardHandler()
{
Enabled.Default = true;
Enabled.Value = true;
}

public override bool Initialize(GameHost host) => true;

public bool HandleKeyEvent(KeyEvent e)
{
if (!Enabled.Value) return false;

if (e.KeyCode == Keycode.Back || e.KeyCode == Keycode.Home || e.KeyCode == Keycode.Menu || e.KeyCode == Keycode.VolumeUp || e.KeyCode == Keycode.VolumeDown || e.KeyCode == Keycode.VolumeMute)
return false;

if ((e.Source & InputSourceType.Keyboard) != InputSourceType.Keyboard)
return false;

var key = mapKey(e.KeyCode);
if (key == Key.Unknown) return false;

bool isDown = e.Action == KeyEventActions.Down;
if (e.RepeatCount > 0 && isDown) return true;

PendingInputs.Enqueue(new KeyboardKeyInput(key, isDown));
return true;
}

private Key mapKey(Keycode code)
{
switch (code)
{
case Keycode.A: return Key.A;
case Keycode.B: return Key.B;
case Keycode.C: return Key.C;
case Keycode.D: return Key.D;
case Keycode.E: return Key.E;
case Keycode.F: return Key.F;
case Keycode.G: return Key.G;
case Keycode.H: return Key.H;
case Keycode.I: return Key.I;
case Keycode.J: return Key.J;
case Keycode.K: return Key.K;
case Keycode.L: return Key.L;
case Keycode.M: return Key.M;
case Keycode.N: return Key.N;
case Keycode.O: return Key.O;
case Keycode.P: return Key.P;
case Keycode.Q: return Key.Q;
case Keycode.R: return Key.R;
case Keycode.S: return Key.S;
case Keycode.T: return Key.T;
case Keycode.U: return Key.U;
case Keycode.V: return Key.V;
case Keycode.W: return Key.W;
case Keycode.X: return Key.X;
case Keycode.Y: return Key.Y;
case Keycode.Z: return Key.Z;
case Keycode.Num0: return Key.Number0;
case Keycode.Num1: return Key.Number1;
case Keycode.Num2: return Key.Number2;
case Keycode.Num3: return Key.Number3;
case Keycode.Num4: return Key.Number4;
case Keycode.Num5: return Key.Number5;
case Keycode.Num6: return Key.Number6;
case Keycode.Num7: return Key.Number7;
case Keycode.Num8: return Key.Number8;
case Keycode.Num9: return Key.Number9;
case Keycode.DpadUp: return Key.Up;
case Keycode.DpadDown: return Key.Down;
case Keycode.DpadLeft: return Key.Left;
case Keycode.DpadRight: return Key.Right;
case Keycode.Enter: return Key.Enter;
case Keycode.Escape: return Key.Escape;
case Keycode.Space: return Key.Space;
case Keycode.Tab: return Key.Tab;
case Keycode.Del: return Key.BackSpace;
case Keycode.ForwardDel: return Key.Delete;
case Keycode.MoveHome: return Key.Home;
case Keycode.MoveEnd: return Key.End;
case Keycode.PageUp: return Key.PageUp;
case Keycode.PageDown: return Key.PageDown;
case Keycode.ShiftLeft: return Key.ShiftLeft;
case Keycode.ShiftRight: return Key.ShiftRight;
case Keycode.CtrlLeft: return Key.ControlLeft;
case Keycode.CtrlRight: return Key.ControlRight;
case Keycode.AltLeft: return Key.AltLeft;
case Keycode.AltRight: return Key.AltRight;
case Keycode.CapsLock: return Key.CapsLock;
case Keycode.F1: return Key.F1;
case Keycode.F2: return Key.F2;
case Keycode.F3: return Key.F3;
case Keycode.F4: return Key.F4;
case Keycode.F5: return Key.F5;
case Keycode.F6: return Key.F6;
case Keycode.F7: return Key.F7;
case Keycode.F8: return Key.F8;
case Keycode.F9: return Key.F9;
case Keycode.F10: return Key.F10;
case Keycode.F11: return Key.F11;
case Keycode.F12: return Key.F12;
case Keycode.Grave: return Key.Tilde;
case Keycode.Minus: return Key.Minus;
case Keycode.Equals: return Key.Plus;
case Keycode.LeftBracket: return Key.BracketLeft;
case Keycode.RightBracket: return Key.BracketRight;
case Keycode.Backslash: return Key.BackSlash;
case Keycode.Semicolon: return Key.Semicolon;
case Keycode.Apostrophe: return Key.Quote;
case Keycode.Comma: return Key.Comma;
case Keycode.Period: return Key.Period;
case Keycode.Slash: return Key.Slash;
default: return Key.Unknown;
}
}
}
}
71 changes: 71 additions & 0 deletions osu.Android/Input/AndroidMouseHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,71 @@
// Copyright (c) ppy Pty Ltd <contact@ppy.sh>. Licensed under the MIT Licence.
// See the LICENCE file in the repository root for full licence text.

using Android.Views;
using osu.Framework.Input.Handlers;
using osu.Framework.Input.StateChanges;
using osu.Framework.Platform;
using osuTK;
using osuTK.Input;

namespace osu.Android.Input
{
public class AndroidMouseHandler : InputHandler
{
public override string Description => "Mouse (Low Latency)";
public override bool IsActive => Enabled.Value;

public AndroidMouseHandler()
{
Enabled.Default = true;
Enabled.Value = true;
}

public override bool Initialize(GameHost host) => true;

public void HandleMotionEvent(MotionEvent e)
{
if (!Enabled.Value) return;

if (e.ActionMasked == MotionEventActions.Scroll)
{
float scrollX = e.GetAxisValue(Axis.Hscroll);
float scrollY = e.GetAxisValue(Axis.Vscroll);
if (scrollX != 0 || scrollY != 0)
{
PendingInputs.Enqueue(new MouseScrollRelativeInput { Delta = new Vector2(scrollX, scrollY), IsPrecise = true });
}
return;
}

for (int i = 0; i < e.HistorySize; i++)
{
handlePointer(e, i);
}
handlePointer(e, -1);
}

private void handlePointer(MotionEvent e, int historyIndex)
{
const int pointer_index = 0;
if (e.PointerCount <= pointer_index) return;

float x = historyIndex < 0 ? e.GetX(pointer_index) : e.GetHistoricalX(pointer_index, historyIndex);
float y = historyIndex < 0 ? e.GetY(pointer_index) : e.GetHistoricalY(pointer_index, historyIndex);

PendingInputs.Enqueue(new MousePositionAbsoluteInput { Position = new Vector2(x, y) });

bool left = (e.ButtonState & MotionEventButtonState.Primary) != 0;
bool right = (e.ButtonState & MotionEventButtonState.Secondary) != 0;
bool middle = (e.ButtonState & MotionEventButtonState.Tertiary) != 0;

if (left != lastLeft) { PendingInputs.Enqueue(new MouseButtonInput(MouseButton.Left, left)); lastLeft = left; }
if (right != lastRight) { PendingInputs.Enqueue(new MouseButtonInput(MouseButton.Right, right)); lastRight = right; }
if (middle != lastMiddle) { PendingInputs.Enqueue(new MouseButtonInput(MouseButton.Middle, middle)); lastMiddle = middle; }
}

private bool lastLeft;
private bool lastRight;
private bool lastMiddle;
}
}
31 changes: 13 additions & 18 deletions osu.Android/Input/AndroidStylusHandler.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,36 +8,32 @@
using osu.Framework.Input.Handlers.Tablet;
using osu.Framework.Input.StateChanges;
using osu.Framework.Platform;
using osu.Framework.Logging;
using osuTK;
using osuTK.Input;

namespace osu.Android.Input
{
public class AndroidStylusHandler : InputHandler, ITabletHandler
{
public override string Description => "S Pen / Stylus";
public override string Description => "S Pen / Stylus (Low Latency)";
public override bool IsActive => Enabled.Value;

public Bindable<Vector2> AreaOffset { get; } = new Bindable<Vector2>();
public Bindable<Vector2> AreaSize { get; } = new Bindable<Vector2>();
public Bindable<Vector2> OutputAreaSize { get; } = new Bindable<Vector2>();
public Bindable<Vector2> OutputAreaOffset { get; } = new Bindable<Vector2>();
public IBindable<TabletInfo?> Tablet => tablet;
public Bindable<float> Rotation { get; } = new Bindable<float>();
public BindableFloat PressureThreshold { get; } = new BindableFloat(0.05f)
public BindableFloat PressureThreshold { get; } = new BindableFloat(0.1f)
{
MinValue = 0f,
MaxValue = 1f,
Precision = 0.005f,
MinValue = 0.01f,
MaxValue = 0.9f,
};

private readonly Bindable<TabletInfo?> tablet = new Bindable<TabletInfo?>();

public override bool IsActive => Enabled.Value;

private bool lastLeftDown;
private bool lastRightDown;
private bool firstEventReceived;

public AndroidStylusHandler()
{
Expand All @@ -47,7 +43,6 @@ public AndroidStylusHandler()

public override bool Initialize(GameHost host)
{
// Initial tablet info with a sane default. We'll refine this as events arrive.
tablet.Value = new TabletInfo("S Pen", new Vector2(2000, 1000));
return base.Initialize(host);
}
Expand All @@ -56,13 +51,16 @@ public void HandleMotionEvent(MotionEvent e)
{
if (!Enabled.Value) return;

if (!firstEventReceived)
// Handle hover entry/exit separately if needed, but for now we focus on position and buttons.
if (e.ActionMasked == MotionEventActions.HoverExit || e.ActionMasked == MotionEventActions.Up || e.ActionMasked == MotionEventActions.Cancel)
{
Logger.Log($"[osu!] S Pen input detected. Source={e.Source}, ToolType={e.GetToolType(0)}", LoggingTarget.Input);
firstEventReceived = true;
if (lastLeftDown) { PendingInputs.Enqueue(new MouseButtonInput(MouseButton.Left, false)); lastLeftDown = false; }
if (lastRightDown) { PendingInputs.Enqueue(new MouseButtonInput(MouseButton.Right, false)); lastRightDown = false; }

if (e.ActionMasked != MotionEventActions.HoverExit)
return;
}

// Process historical points for maximum accuracy.
for (int i = 0; i < e.HistorySize; i++)
{
handlePointer(e, i);
Expand All @@ -73,31 +71,28 @@ public void HandleMotionEvent(MotionEvent e)
private void handlePointer(MotionEvent e, int historyIndex)
{
const int pointer_index = 0;
if (e.PointerCount <= pointer_index) return;

float x = historyIndex < 0 ? e.GetX(pointer_index) : e.GetHistoricalX(pointer_index, historyIndex);
float y = historyIndex < 0 ? e.GetY(pointer_index) : e.GetHistoricalY(pointer_index, historyIndex);
float pressure = historyIndex < 0 ? e.GetPressure(pointer_index) : e.GetHistoricalPressure(pointer_index, historyIndex);

// Dynamically update tablet bounds.
if (tablet.Value == null || x > tablet.Value.Size.X || y > tablet.Value.Size.Y)
{
var currentSize = tablet.Value?.Size ?? Vector2.Zero;
var newSize = new Vector2(Math.Max(x, currentSize.X), Math.Max(y, currentSize.Y));
tablet.Value = new TabletInfo("S Pen", newSize);
}

// Report absolute position.
PendingInputs.Enqueue(new MousePositionAbsoluteInput { Position = new Vector2(x, y) });

// Map pressure to mouse buttons.
bool isLeftDown = pressure >= PressureThreshold.Value;
if (isLeftDown != lastLeftDown)
{
PendingInputs.Enqueue(new MouseButtonInput(MouseButton.Left, isLeftDown));
lastLeftDown = isLeftDown;
}

// Map side button to Right Click.
bool isRightDown = (e.ButtonState & MotionEventButtonState.StylusPrimary) != 0;
if (isRightDown != lastRightDown)
{
Expand Down
52 changes: 52 additions & 0 deletions osu.Android/OsuGameActivity.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,8 @@ public class OsuGameActivity : AndroidGameActivity, ISurfaceHolderCallback

public new bool IsTablet { get; private set; }
internal AndroidStylusHandler? StylusHandler;
internal AndroidKeyboardHandler? KeyboardHandler;
internal AndroidMouseHandler? MouseHandler;

private OsuGameAndroid? game;

Expand Down Expand Up @@ -123,6 +125,56 @@ protected override void OnCreate(Bundle? savedInstanceState)

protected override void OnNewIntent(Intent? intent) => handleIntent(intent);

public override bool DispatchKeyEvent(KeyEvent? e)
{
if (e != null && KeyboardHandler != null && KeyboardHandler.HandleKeyEvent(e))
return true;

return base.DispatchKeyEvent(e);
}

public override bool DispatchTouchEvent(MotionEvent? e)
{
if (e == null) return base.DispatchTouchEvent(e);

if (isStylusEvent(e))
{
if (e.ActionMasked == MotionEventActions.Down || e.ActionMasked == MotionEventActions.HoverEnter) Window?.DecorView.RequestUnbufferedDispatch(e);
StylusHandler?.HandleMotionEvent(e);
return true;
}

if ((e.Source & InputSourceType.Mouse) == InputSourceType.Mouse)
{
if (e.ActionMasked == MotionEventActions.Down) Window?.DecorView.RequestUnbufferedDispatch(e);
MouseHandler?.HandleMotionEvent(e);
return true;
}

return base.DispatchTouchEvent(e);
}

public override bool DispatchGenericMotionEvent(MotionEvent? e)
{
if (e == null) return base.DispatchGenericMotionEvent(e);

if (isStylusEvent(e))
{
if (e.ActionMasked == MotionEventActions.Down || e.ActionMasked == MotionEventActions.HoverEnter) Window?.DecorView.RequestUnbufferedDispatch(e);
StylusHandler?.HandleMotionEvent(e);
return true;
}

if ((e.Source & InputSourceType.Mouse) == InputSourceType.Mouse)
{
if (e.ActionMasked == MotionEventActions.Down) Window?.DecorView.RequestUnbufferedDispatch(e);
MouseHandler?.HandleMotionEvent(e);
return true;
}

return base.DispatchGenericMotionEvent(e);
}

public override bool OnTouchEvent(MotionEvent? e)
{
if (e != null && isStylusEvent(e))
Expand Down
10 changes: 10 additions & 0 deletions osu.Android/OsuGameAndroid.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -126,6 +126,8 @@ public override Version AssemblyVersion
}

private AndroidStylusHandler? stylusHandler;
private AndroidMouseHandler? mouseHandler;
private AndroidKeyboardHandler? keyboardHandler;

[BackgroundDependencyLoader]
private void load()
Expand All @@ -139,6 +141,14 @@ private void load()
Host.AvailableInputHandlers.Add(stylusHandler);
gameActivity.StylusHandler = stylusHandler;

mouseHandler = new AndroidMouseHandler();
Host.AvailableInputHandlers.Add(mouseHandler);
gameActivity.MouseHandler = mouseHandler;

keyboardHandler = new AndroidKeyboardHandler();
Host.AvailableInputHandlers.Add(keyboardHandler);
gameActivity.KeyboardHandler = keyboardHandler;

startVulkanProbe();

audioRedirector = new OboeAudioRedirector(Audio);
Expand Down
Loading