Virtual/on-screen joystick support (#2803)

* Virtual/on-screen gamepad

* documentation

* disable AndroidSensorJoyInput

* reduce objects allocations

* fixes

* fixes autoshow

* improved layouts

Author: riccardobl

jme3-android-examples/src/main/java/jme3test/android/TestAndroidSensors.java

@@ -17,6 +17,7 @@
 import com.jme3.scene.Mesh;
 import com.jme3.scene.shape.Box;
 import com.jme3.scene.shape.Line;
+import com.jme3.system.AppSettings;
 import com.jme3.texture.Texture;
 import com.jme3.util.IntMap;
 
@@ -79,6 +80,12 @@ public class TestAndroidSensors extends SimpleApplication implements ActionListe
 
     // Make sure to set joystickEventsEnabled = true in MainActivity for Android
 
+    public static void configureSettings(AppSettings settings) {
+        settings.setUseJoysticks(true);
+        settings.setUseAndroidSensorJoystick(true);
+        settings.setVirtualJoystick(AppSettings.VIRTUAL_JOYSTICK_DISABLED);
+    }
+
     private float toDegrees(float rad) {
         return rad * FastMath.RAD_TO_DEG;
     }
@@ -311,4 +318,4 @@ public void onAnalog(String string, float value, float tpf) {
         }
 
     }
-}
+}

jme3-android/src/main/java/com/jme3/app/AndroidHarnessFragment.java

@@ -42,7 +42,7 @@
 import androidx.fragment.app.Fragment;
 import com.jme3.audio.AudioRenderer;
 import com.jme3.input.JoyInput;
-import com.jme3.input.android.AndroidSensorJoyInput;
+import com.jme3.input.android.AndroidJoyInput;
 import com.jme3.system.AppSettings;
 import com.jme3.system.SystemListener;
 import com.jme3.system.android.JmeAndroidSystem;
@@ -269,8 +269,8 @@ public void gainFocus() {
             }
 
             JoyInput joyInput = app.getContext() != null ? app.getContext().getJoyInput() : null;
-            if (joyInput instanceof AndroidSensorJoyInput) {
-                ((AndroidSensorJoyInput) joyInput).resumeSensors();
+            if (joyInput instanceof AndroidJoyInput) {
+                ((AndroidJoyInput) joyInput).resumeJoysticks();
             }
 
             app.gainFocus();
@@ -295,8 +295,8 @@ public void loseFocus() {
             }
 
             JoyInput joyInput = app.getContext() != null ? app.getContext().getJoyInput() : null;
-            if (joyInput instanceof AndroidSensorJoyInput) {
-                ((AndroidSensorJoyInput) joyInput).pauseSensors();
+            if (joyInput instanceof AndroidJoyInput) {
+                ((AndroidJoyInput) joyInput).pauseJoysticks();
             }
         }
     }

jme3-android/src/main/java/com/jme3/input/android/AndroidInputHandler.java

@@ -208,6 +208,10 @@ public boolean onTouch(View view, MotionEvent event) {
 //        logger.log(Level.INFO, "onTouch source: {0}, isTouch: {1}",
 //                new Object[]{source, isTouch});
 
+        if (isTouch && joyInput != null && joyInput.onTouch(event)) {
+            return true;
+        }
+
         if (isTouch && touchInput != null) {
             // send the event to the touch processor
             consumed = touchInput.onTouch(event);
@@ -234,6 +238,10 @@ public boolean onKey(View view, int keyCode, KeyEvent event) {
 //        logger.log(Level.INFO, "onKey source: {0}, isTouch: {1}",
 //                new Object[]{source, isTouch});
 
+        if ((source & InputDevice.SOURCE_KEYBOARD) == InputDevice.SOURCE_KEYBOARD && joyInput != null) {
+            joyInput.onKeyboardInput();
+        }
+
         if (touchInput != null) {
             consumed = touchInput.onKey(event);
         }

jme3-android/src/main/java/com/jme3/input/android/AndroidInputHandler14.java

@@ -179,6 +179,10 @@ public boolean onKey(View view, int keyCode, KeyEvent event) {
         boolean isUnknown =
                 (source & android.view.InputDevice.SOURCE_UNKNOWN) == android.view.InputDevice.SOURCE_UNKNOWN;
 
+        if ((source & InputDevice.SOURCE_KEYBOARD) == InputDevice.SOURCE_KEYBOARD && joyInput != null) {
+            joyInput.onKeyboardInput();
+        }
+
         if (touchInput != null && (isTouch || (isUnknown && this.touchInput.isSimulateKeyboard()))) {
 //            logger.log(Level.INFO, "onKey source: {0}, isTouch: {1}",
 //                    new Object[]{source, isTouch});

jme3-android/src/main/java/com/jme3/input/android/AndroidJoyInput.java

@@ -32,13 +32,15 @@
 package com.jme3.input.android;
 
 import android.opengl.GLSurfaceView;
+import android.view.MotionEvent;
 import com.jme3.input.InputManager;
 import com.jme3.input.JoyInput;
 import com.jme3.input.Joystick;
 import com.jme3.input.RawInputListener;
 import com.jme3.input.event.InputEvent;
 import com.jme3.input.event.JoyAxisEvent;
 import com.jme3.input.event.JoyButtonEvent;
+import com.jme3.input.virtual.VirtualJoystick;
 import com.jme3.system.AppSettings;
 import com.jme3.system.JmeSystem;
 import java.util.ArrayList;
@@ -48,9 +50,8 @@
 import java.util.logging.Logger;
 
 /**
- * Main class that manages various joystick devices.  Joysticks can be many forms
- * including a simulated joystick to communicate the device orientation as well
- * as physical gamepads. <br>
+ * Main class that manages joystick devices. Joysticks can be physical gamepads,
+ * the on-screen virtual joystick, or an explicitly-enabled sensor joystick. <br>
  * This class manages all the joysticks and feeds the inputs from each back
  * to jME's InputManager.
  *
@@ -67,7 +68,7 @@
  *
  * MainActivity needs the following line to enable Joysticks on Android platforms
  *    joystickEventsEnabled = true;
- * This is done to allow for battery conservation when sensor data or gamepads
+ * This is done to allow for battery conservation when sensor data or joysticks
  * are not required by the application.
  *
  * {@code
@@ -79,7 +80,6 @@
  */
 public class AndroidJoyInput implements JoyInput {
     private static final Logger logger = Logger.getLogger(AndroidJoyInput.class.getName());
-    public static boolean disableSensors = false;
 
     protected AndroidInputHandler inputHandler;
     protected List<Joystick> joystickList = new ArrayList<>();
@@ -92,20 +92,32 @@ public class AndroidJoyInput implements JoyInput {
     private ConcurrentLinkedQueue<InputEvent> eventQueue = new ConcurrentLinkedQueue<>();
     private AndroidSensorJoyInput sensorJoyInput;
     private boolean onDeviceJoystickRumble = false;
+    private String virtualJoystickMode = AppSettings.VIRTUAL_JOYSTICK_AUTO;
+    private String virtualJoystickDefaultLayout = AppSettings.VIRTUAL_JOYSTICK_LAYOUT_DYNAMIC;
+    private boolean useJoysticks = true;
+    private boolean useAndroidSensorJoystick = false;
+    private boolean physicalJoystickAvailable = false;
+    private boolean keyboardSuppressedAutoJoystick = false;
+    private volatile VirtualJoystick virtualJoystick;
+    private GLSurfaceView view;
 
     public AndroidJoyInput(AndroidInputHandler inputHandler) {
         this.inputHandler = inputHandler;
-        sensorJoyInput = new AndroidSensorJoyInput(this);
     }
 
     public void setView(GLSurfaceView view) {
+        this.view = view;
         if (sensorJoyInput != null) {
             sensorJoyInput.setView(view);
         }
     }
 
     public void loadSettings(AppSettings settings) {
         onDeviceJoystickRumble = settings.isOnDeviceJoystickRumble();
+        virtualJoystickMode = settings.getVirtualJoystickMode();
+        virtualJoystickDefaultLayout = settings.getVirtualJoystickDefaultLayout();
+        useJoysticks = settings.useJoysticks();
+        useAndroidSensorJoystick = settings.useAndroidSensorJoystick();
     }
 
     boolean isOnDeviceJoystickRumble() {
@@ -127,6 +139,9 @@ public void pauseJoysticks() {
         if (onDeviceJoystickRumble) {
             JmeSystem.stopRumble();
         }
+        if (virtualJoystick != null) {
+            virtualJoystick.onPointerCancel(0L);
+        }
 
     }
 
@@ -138,7 +153,6 @@ public void resumeJoysticks() {
         if (sensorJoyInput != null) {
             sensorJoyInput.resumeSensors();
         }
-
     }
 
     @Override
@@ -154,12 +168,11 @@ public boolean isInitialized() {
     @Override
     public void destroy() {
         initialized = false;
-
         if (sensorJoyInput != null) {
             sensorJoyInput.destroy();
+            sensorJoyInput = null;
         }
-
-        setView(null);
+        view = null;
     }
 
     @Override
@@ -191,12 +204,72 @@ public Joystick[] loadJoysticks(InputManager inputManager) {
         if (logger.isLoggable(Level.INFO)) {
             logger.log(Level.INFO, "loading joysticks for {0}", this.getClass().getName());
         }
-        if (!disableSensors) {
+        joystickList.clear();
+        if (useJoysticks && useAndroidSensorJoystick) {
+            if (sensorJoyInput == null) {
+                sensorJoyInput = new AndroidSensorJoyInput(this);
+                sensorJoyInput.setView(view);
+            }
             joystickList.add(sensorJoyInput.loadJoystick(joystickList.size(), inputManager));
         }
+        physicalJoystickAvailable = false;
+        if (shouldCreateVirtualJoystick()) {
+            virtualJoystick = new VirtualJoystick(inputManager, this, joystickList.size());
+            virtualJoystick.setLayout(VirtualJoystick.createLayout(virtualJoystickDefaultLayout));
+            virtualJoystick.setEnabled(false);
+            updateVirtualJoystickAutoVisibility();
+            joystickList.add(virtualJoystick);
+        } else {
+            virtualJoystick = null;
+        }
         return joystickList.toArray( new Joystick[joystickList.size()] );
     }
 
+    public boolean onTouch(MotionEvent event) {
+        VirtualJoystick joystick = virtualJoystick;
+        if (joystick == null || inputHandler.getView() == null) {
+            return false;
+        }
+
+        boolean consumed = false;
+        int action = event.getAction() & MotionEvent.ACTION_MASK;
+        int pointerIndex = (event.getAction() & MotionEvent.ACTION_POINTER_INDEX_MASK)
+                >> MotionEvent.ACTION_POINTER_INDEX_SHIFT;
+        long time = event.getEventTime();
+
+        switch (action) {
+            case MotionEvent.ACTION_POINTER_DOWN:
+            case MotionEvent.ACTION_DOWN:
+                consumed = joystick.onPointerDown(event.getPointerId(pointerIndex),
+                        toJmeX(event.getX(pointerIndex)), toJmeY(event.getY(pointerIndex)), time);
+                break;
+            case MotionEvent.ACTION_POINTER_UP:
+            case MotionEvent.ACTION_UP:
+                consumed = joystick.onPointerUp(event.getPointerId(pointerIndex),
+                        toJmeX(event.getX(pointerIndex)), toJmeY(event.getY(pointerIndex)), time);
+                break;
+            case MotionEvent.ACTION_CANCEL:
+                consumed = joystick.onPointerCancel(time);
+                break;
+            case MotionEvent.ACTION_MOVE:
+                for (int i = 0; i < event.getPointerCount(); i++) {
+                    consumed = joystick.onPointerMove(event.getPointerId(i),
+                            toJmeX(event.getX(i)), toJmeY(event.getY(i)), time) || consumed;
+                }
+                break;
+            default:
+                break;
+        }
+        return consumed;
+    }
+
+    public void onKeyboardInput() {
+        if (AppSettings.VIRTUAL_JOYSTICK_AUTO.equals(virtualJoystickMode)) {
+            keyboardSuppressedAutoJoystick = true;
+            updateVirtualJoystickAutoVisibility();
+        }
+    }
+
     @Override
     public void update() {
         if (sensorJoyInput != null) {
@@ -214,7 +287,43 @@ public void update() {
                 }
             }
         }
+        if (virtualJoystick != null) {
+            updateVirtualJoystickAutoVisibility();
+            virtualJoystick.dispatchEvents(listener);
+        }
+
+    }
+
+    private float toJmeX(float x) {
+        return inputHandler.touchInput.getJmeX(x);
+    }
+
+    private float toJmeY(float y) {
+        return inputHandler.touchInput.invertY(inputHandler.touchInput.getJmeY(y));
+    }
+
+    protected void setPhysicalJoystickAvailable(boolean available) {
+        physicalJoystickAvailable = available;
+        updateVirtualJoystickAutoVisibility();
+    }
 
+    private boolean shouldCreateVirtualJoystick() {
+        return useJoysticks && !AppSettings.VIRTUAL_JOYSTICK_DISABLED.equals(virtualJoystickMode);
+    }
+
+    private void updateVirtualJoystickAutoVisibility() {
+        if (virtualJoystick == null) {
+            return;
+        }
+        boolean wasEnabled = virtualJoystick.isEnabled();
+        boolean active = AppSettings.VIRTUAL_JOYSTICK_ENABLED.equals(virtualJoystickMode)
+                || (AppSettings.VIRTUAL_JOYSTICK_AUTO.equals(virtualJoystickMode)
+                && !physicalJoystickAvailable
+                && !keyboardSuppressedAutoJoystick
+                && virtualJoystick.hasInputBindings());
+        if (wasEnabled != active) {
+            virtualJoystick.setEnabled(active);
+        }
     }
 
 }

jme3-android/src/main/java/com/jme3/input/android/AndroidJoyInput14.java

@@ -90,10 +90,12 @@ public void destroy() {
 
     @Override
     public Joystick[] loadJoysticks(InputManager inputManager) {
-        // load the simulated joystick for device orientation
+        // load virtual joystick if enabled
         super.loadJoysticks(inputManager);
         // load physical gamepads/joysticks
+        int beforePhysicalJoysticks = joystickList.size();
         joystickList.addAll(joystickJoyInput.loadJoysticks(joystickList.size(), inputManager));
+        setPhysicalJoystickAvailable(joystickList.size() > beforePhysicalJoysticks);
         // return the list of joysticks back to InputManager
         return joystickList.toArray( new Joystick[joystickList.size()] );
     }

jme3-core/src/main/java/com/jme3/app/SimpleApplication.java

@@ -37,10 +37,13 @@
 import com.jme3.font.BitmapFont;
 import com.jme3.font.BitmapText;
 import com.jme3.input.FlyByCamera;
+import com.jme3.input.Joystick;
 import com.jme3.input.KeyInput;
 import com.jme3.input.controls.ActionListener;
 import com.jme3.input.controls.KeyTrigger;
+import com.jme3.input.virtual.VirtualJoystick;
 import com.jme3.profile.AppStep;
+import com.jme3.renderer.Camera;
 import com.jme3.renderer.RenderManager;
 import com.jme3.renderer.queue.RenderQueue.Bucket;
 import com.jme3.scene.Node;
@@ -336,6 +339,7 @@ public void update() {
         if (prof != null) {
             prof.appStep(AppStep.SpatialUpdate);
         }
+        updateVirtualJoystickVisuals(tpf);
         rootNode.updateLogicalState(tpf);
         guiNode.updateLogicalState(tpf);
 
@@ -410,4 +414,22 @@ public void simpleUpdate(float tpf) {
     public void simpleRender(RenderManager rm) {
         // Default empty implementation; subclasses can override
     }
+
+    private void updateVirtualJoystickVisuals(float tpf) {
+        if (inputManager == null || assetManager == null || guiViewPort == null) {
+            return;
+        }
+        Joystick[] joysticks = inputManager.getJoysticks();
+        if (joysticks == null) {
+            return;
+        }
+        Camera guiCamera = guiViewPort.getCamera();
+        int width = guiCamera != null ? guiCamera.getWidth() : settings.getWidth();
+        int height = guiCamera != null ? guiCamera.getHeight() : settings.getHeight();
+        for (Joystick joystick : joysticks) {
+            if (joystick instanceof VirtualJoystick) {
+                ((VirtualJoystick) joystick).updateVisuals(guiNode, assetManager, width, height, tpf);
+            }
+        }
+    }
 }