Skip to content

InputService: real-time gestures #285

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
gm-vm wants to merge 4 commits into from
Closed

InputService: real-time gestures #285

Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
7882167
InputService: real-time gestures
gm-vm Mar 30, 2025
5668877
Keep a GestureHandler per InputContext
gm-vm Mar 30, 2025
762fe2b
Remove some unneeded changes
gm-vm Mar 30, 2025
0016ddc
Minor cleanup
gm-vm Mar 30, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
160 changes: 128 additions & 32 deletions app/src/main/java/net/christianbeier/droidvnc_ng/InputService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,7 @@ public synchronized void onCancelled(GestureDescription gestureDescription) {
private static class InputContext {
// pointer-related
boolean isButtonOneDown;
Path path = new Path();
long lastGestureStartTime;
GestureHandler gestureHandler;
GestureCallback gestureCallback = new GestureCallback();
InputPointerView pointerView;
// keyboard-related
Expand All @@ -81,6 +80,14 @@ private static class InputContext {

private int displayId;

InputContext() {
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O) {
gestureHandler = new ProgressiveGestureHandler(instance, this);
} else {
gestureHandler = new LegacyGestureHandler(instance, this);
}
}

int getDisplayId() {return displayId;}

/**
Expand Down Expand Up @@ -194,6 +201,7 @@ public static void addClient(long client, boolean withPointer) {
int displayId = Display.DEFAULT_DISPLAY;
InputContext inputContext = new InputContext();
inputContext.setDisplayId(displayId);

if(withPointer) {
inputContext.pointerView = new InputPointerView(
instance,
Expand Down Expand Up @@ -260,18 +268,18 @@ public static void onPointerEvent(int buttonMask, int x, int y, long client) {
// down, was up
if ((buttonMask & (1 << 0)) != 0 && !inputContext.isButtonOneDown) {
inputContext.isButtonOneDown = true;
instance.startGesture(inputContext, x, y);
Copy link
Owner

@bk138 bk138 Mar 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This introduces a regression regarding multi-client input. Please leave the logic client-specific.

Copy link
Author

@gm-vm gm-vm Mar 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Right, I was so focused on getting gestures to work that I didn't even notice that there are multiple InputContext instances. Fixed this here.

inputContext.gestureHandler.startGesture(x, y);
}

// down, was down
if ((buttonMask & (1 << 0)) != 0 && inputContext.isButtonOneDown) {
instance.continueGesture(inputContext, x, y);
inputContext.gestureHandler.continueGesture(x, y);
}

// up, was down
if ((buttonMask & (1 << 0)) == 0 && inputContext.isButtonOneDown) {
inputContext.isButtonOneDown = false;
instance.endGesture(inputContext, x, y);
inputContext.gestureHandler.endGesture(x, y);
}


Expand Down Expand Up @@ -786,33 +794,6 @@ public static boolean isTakingScreenShots() {
}
}

private void startGesture(InputContext inputContext, int x, int y) {
inputContext.path.reset();
inputContext.path.moveTo( x, y );
inputContext.lastGestureStartTime = System.currentTimeMillis();
}

private void continueGesture(InputContext inputContext, int x, int y) {
inputContext.path.lineTo( x, y );
}

private void endGesture(InputContext inputContext, int x, int y) {
inputContext.path.lineTo( x, y );
long duration = System.currentTimeMillis() - inputContext.lastGestureStartTime;
// gesture ended very very shortly after start (< 1ms). make it 1ms to get dispatched to the system
if (duration == 0) duration = 1;
GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription( inputContext.path, 0, duration);
GestureDescription.Builder builder = new GestureDescription.Builder();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
builder.setDisplayId(inputContext.getDisplayId());
}
builder.addStroke(stroke);
// Docs says: Any gestures currently in progress, whether from the user, this service, or another service, will be cancelled.
// But at least on API level 32, setting different display ids with the builder allows for parallel input.
dispatchGesture(builder.build(), null, null);
}


private void longPress(InputContext inputContext, int x, int y )
{
dispatchGesture( createClick(inputContext, x, y, ViewConfiguration.getTapTimeout() + ViewConfiguration.getLongPressTimeout()), null, null );
Expand Down Expand Up @@ -879,4 +860,119 @@ private static void setCursorPos(AccessibilityNodeInfo node, int cursorPos) {
action.putInt(AccessibilityNodeInfo.ACTION_ARGUMENT_SELECTION_END_INT, cursorPos);
node.performAction(AccessibilityNodeInfo.AccessibilityAction.ACTION_SET_SELECTION.getId(), action);
}

private interface GestureHandler {
void startGesture(int x, int y);
void continueGesture(int x, int y);
void endGesture(int x, int y);
}

private static class LegacyGestureHandler implements GestureHandler {
Copy link
Owner

@bk138 bk138 Mar 30, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

no need for these classes IMO, just do the Android version differentiation in the existing handlers .-)

Copy link
Author

@gm-vm gm-vm Mar 30, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

What did you have in mind exactly? I'm trying to merge the two handlers, but the code is getting a lot more confusing and hard to follow.

Unrelated: I noticed that you were using System.currentTimeMillis(), which is not monotonic. This means that if the system time changes while there is an ongoing gesture the resulting duration will be wrong. SystemClock.elapsedRealtime() is better for measuring intervals.

private final AccessibilityService service;
private final InputContext inputContext;
private final Path path = new Path();
private long lastGestureStartTime;

private LegacyGestureHandler(AccessibilityService service, InputContext inputContext) {
this.service = service;
this.inputContext = inputContext;
}

@Override
public void startGesture(int x, int y) {
path.reset();
path.moveTo( x, y );
lastGestureStartTime = System.currentTimeMillis();
}

@Override
public void continueGesture(int x, int y) {
path.lineTo( x, y );
}

@Override
public void endGesture(int x, int y) {
path.lineTo( x, y );
long duration = System.currentTimeMillis() - lastGestureStartTime;
// gesture ended very very shortly after start (< 1ms). make it 1ms to get dispatched to the system
if (duration == 0) duration = 1;
GestureDescription.StrokeDescription stroke = new GestureDescription.StrokeDescription( path, 0, duration);
GestureDescription.Builder builder = new GestureDescription.Builder();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
builder.setDisplayId(inputContext.getDisplayId());
}
builder.addStroke(stroke);
// Docs says: Any gestures currently in progress, whether from the user, this service, or another service, will be cancelled.
// But at least on API level 32, setting different display ids with the builder allows for parallel input.
service.dispatchGesture(builder.build(), null, null);
}
}

@RequiresApi(api = Build.VERSION_CODES.O)
private static class ProgressiveGestureHandler implements GestureHandler {
private final AccessibilityService service;
private final InputContext inputContext;
private final Path currentPath = new Path();
private GestureDescription.StrokeDescription currentStroke;
private long lastDispatch;

@RequiresApi(api = Build.VERSION_CODES.O)
public ProgressiveGestureHandler(AccessibilityService service, InputContext inputContext) {
this.service = service;
this.inputContext = inputContext;
}

@Override
public void startGesture(int x, int y) {
lastDispatch = SystemClock.elapsedRealtime();
currentPath.reset();
currentPath.moveTo(x, y);
}

@Override
public void continueGesture(int x, int y) {
dispatch(x, y, true);
currentPath.reset();
currentPath.moveTo(x, y);
}

@Override
public void endGesture(int x, int y) {
dispatch(x, y, false);
currentStroke = null;
}

private void dispatch(int x, int y, boolean willContinue) {
currentPath.lineTo(x, y);
long currentTime = SystemClock.elapsedRealtime();
// gesture ended very very shortly after start (< 1ms). make it 1ms to get dispatched to the system
long duration = Math.max(1, currentTime - lastDispatch);

if (currentStroke == null) {
currentStroke = new GestureDescription.StrokeDescription(
currentPath,
0,
duration,
willContinue
);
} else {
currentStroke = currentStroke.continueStroke(
currentPath,
0,
duration,
willContinue
);
}

GestureDescription.Builder gestureBuilder = new GestureDescription.Builder();
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.R) {
gestureBuilder.setDisplayId(inputContext.getDisplayId());
}
gestureBuilder.addStroke(currentStroke);
// Docs says: Any gestures currently in progress, whether from the user, this service, or another service, will be cancelled.
// But at least on API level 32, setting different display ids with the builder allows for parallel input.
service.dispatchGesture(gestureBuilder.build(), null, null);
lastDispatch = currentTime;
}
}
}