Face detection and position tracking

CapgoCameraPreview.podspec

@@ -14,4 +14,5 @@ Pod::Spec.new do |s|
   s.ios.deployment_target = '15.0'
   s.dependency 'Capacitor'
   s.swift_version = '5.1'
+
 end

README.md

@@ -350,6 +350,7 @@ Documentation for the [uploader](https://github.com/Cap-go/capacitor-uploader)
 * [`setFocus(...)`](#setfocus)
 * [`addListener('screenResize', ...)`](#addlistenerscreenresize-)
 * [`addListener('orientationChange', ...)`](#addlistenerorientationchange-)
+* [`addListener('faceDetection', ...)`](#addlistenerfacedetection-)
 * [`deleteFile(...)`](#deletefile)
 * [`getSafeAreaInsets()`](#getsafeareainsets)
 * [`getOrientation()`](#getorientation)
@@ -360,6 +361,9 @@ Documentation for the [uploader](https://github.com/Cap-go/capacitor-uploader)
 * [`getExposureCompensation()`](#getexposurecompensation)
 * [`setExposureCompensation(...)`](#setexposurecompensation)
 * [`getPluginVersion()`](#getpluginversion)
+* [`startFaceDetection(...)`](#startfacedetection)
+* [`stopFaceDetection()`](#stopfacedetection)
+* [`isFaceDetectionRunning()`](#isfacedetectionrunning)
 * [Interfaces](#interfaces)
 * [Type Aliases](#type-aliases)
 * [Enums](#enums)
@@ -912,6 +916,27 @@ Adds a listener for orientation change events.
 --------------------
 
 
+### addListener('faceDetection', ...)
+
+```typescript
+addListener(eventName: 'faceDetection', listenerFunc: (result: FaceDetectionResult) => void) => Promise<PluginListenerHandle>
+```
+
+Adds a listener for face detection events.
+This event is emitted continuously while face detection is active.
+
+| Param              | Type                                                                                     | Description                                              |
+| ------------------ | ---------------------------------------------------------------------------------------- | -------------------------------------------------------- |
+| **`eventName`**    | <code>'faceDetection'</code>                                                             | - Must be 'faceDetection'                                |
+| **`listenerFunc`** | <code>(result: <a href="#facedetectionresult">FaceDetectionResult</a>) =&gt; void</code> | - Callback function that receives face detection results |
+
+**Returns:** <code>Promise&lt;<a href="#pluginlistenerhandle">PluginListenerHandle</a>&gt;</code>
+
+**Since:** 7.27.0
+
+--------------------
+
+
 ### deleteFile(...)
 
 ```typescript
@@ -1064,6 +1089,53 @@ Get the native Capacitor plugin version
 --------------------
 
 
+### startFaceDetection(...)
+
+```typescript
+startFaceDetection(options?: FaceDetectionOptions | undefined) => Promise<void>
+```
+
+Starts real-time face detection on the camera preview.
+Face detection results will be emitted through the 'faceDetection' event listener.
+
+| Param         | Type                                                                  | Description                                |
+| ------------- | --------------------------------------------------------------------- | ------------------------------------------ |
+| **`options`** | <code><a href="#facedetectionoptions">FaceDetectionOptions</a></code> | - Configuration options for face detection |
+
+**Since:** 7.27.0
+
+--------------------
+
+
+### stopFaceDetection()
+
+```typescript
+stopFaceDetection() => Promise<void>
+```
+
+Stops real-time face detection.
+After calling this, no more 'faceDetection' events will be emitted.
+
+**Since:** 7.27.0
+
+--------------------
+
+
+### isFaceDetectionRunning()
+
+```typescript
+isFaceDetectionRunning() => Promise<{ isDetecting: boolean; }>
+```
+
+Checks if face detection is currently running.
+
+**Returns:** <code>Promise&lt;{ isDetecting: boolean; }&gt;</code>
+
+**Since:** 7.27.0
+
+--------------------
+
+
 ### Interfaces
 
 
@@ -1227,6 +1299,74 @@ Represents the detailed information of the currently active lens.
 | **`remove`** | <code>() =&gt; Promise&lt;void&gt;</code> |
 
 
+#### FaceDetectionResult
+
+Result from face detection containing all detected faces.
+
+| Prop              | Type                        | Description                                                        |
+| ----------------- | --------------------------- | ------------------------------------------------------------------ |
+| **`faces`**       | <code>DetectedFace[]</code> | Array of detected faces. Empty if no faces detected.               |
+| **`frameWidth`**  | <code>number</code>         | Width of the frame in pixels.                                      |
+| **`frameHeight`** | <code>number</code>         | Height of the frame in pixels.                                     |
+| **`timestamp`**   | <code>number</code>         | Timestamp when the frame was processed (milliseconds since epoch). |
+
+
+#### DetectedFace
+
+Represents a single detected face with all its properties.
+
+| Prop             | Type                                                    | Description                                                                                                                                                              |
+| ---------------- | ------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| **`trackingId`** | <code>number</code>                                     | Unique tracking ID for this face (persists across frames when tracking is enabled). Use this to associate face data across multiple frames for smooth filter animations. |
+| **`bounds`**     | <code><a href="#facebounds">FaceBounds</a></code>       | Face bounding box with normalized coordinates (0.0 - 1.0). Coordinates are relative to the camera preview dimensions.                                                    |
+| **`rollAngle`**  | <code>number</code>                                     | Face rotation angle around the Z-axis (head tilt left/right) in degrees. - Negative values: head tilted left - Positive values: head tilted right - Range: -180 to +180  |
+| **`yawAngle`**   | <code>number</code>                                     | Face rotation angle around the Y-axis (head turn left/right) in degrees. - Negative values: head turned left - Positive values: head turned right - Range: -180 to +180  |
+| **`pitchAngle`** | <code>number</code>                                     | Face rotation angle around the X-axis (head nod up/down) in degrees. - Negative values: head looking down - Positive values: head looking up - Range: -180 to +180       |
+| **`landmarks`**  | <code><a href="#facelandmarks">FaceLandmarks</a></code> | Facial landmarks with normalized coordinates. Only available if detectLandmarks is enabled.                                                                              |
+
+
+#### FaceBounds
+
+Face bounding box with normalized coordinates (0.0 - 1.0).
+All coordinates are relative to the camera preview dimensions.
+
+| Prop         | Type                | Description                                                 |
+| ------------ | ------------------- | ----------------------------------------------------------- |
+| **`x`**      | <code>number</code> | X coordinate of the top-left corner (normalized 0.0 - 1.0). |
+| **`y`**      | <code>number</code> | Y coordinate of the top-left corner (normalized 0.0 - 1.0). |
+| **`width`**  | <code>number</code> | Width of the bounding box (normalized 0.0 - 1.0).           |
+| **`height`** | <code>number</code> | Height of the bounding box (normalized 0.0 - 1.0).          |
+
+
+#### FaceLandmarks
+
+Facial landmark points with normalized coordinates (0.0 - 1.0).
+All points are relative to the camera preview dimensions.
+
+| Prop              | Type                                    | Description                                      |
+| ----------------- | --------------------------------------- | ------------------------------------------------ |
+| **`leftEye`**     | <code><a href="#point">Point</a></code> | Left eye center position.                        |
+| **`rightEye`**    | <code><a href="#point">Point</a></code> | Right eye center position.                       |
+| **`noseBase`**    | <code><a href="#point">Point</a></code> | Nose base (bottom) position.                     |
+| **`mouthLeft`**   | <code><a href="#point">Point</a></code> | Left mouth corner position.                      |
+| **`mouthRight`**  | <code><a href="#point">Point</a></code> | Right mouth corner position.                     |
+| **`mouthBottom`** | <code><a href="#point">Point</a></code> | Bottom center of the mouth.                      |
+| **`leftEar`**     | <code><a href="#point">Point</a></code> | Left ear position (may not always be detected).  |
+| **`rightEar`**    | <code><a href="#point">Point</a></code> | Right ear position (may not always be detected). |
+| **`leftCheek`**   | <code><a href="#point">Point</a></code> | Left cheek position.                             |
+| **`rightCheek`**  | <code><a href="#point">Point</a></code> | Right cheek position.                            |
+
+
+#### Point
+
+A 2D point with normalized coordinates (0.0 - 1.0).
+
+| Prop    | Type                | Description                                         |
+| ------- | ------------------- | --------------------------------------------------- |
+| **`x`** | <code>number</code> | X coordinate (normalized 0.0 - 1.0, left to right). |
+| **`y`** | <code>number</code> | Y coordinate (normalized 0.0 - 1.0, top to bottom). |
+
+
 #### SafeAreaInsets
 
 Represents safe area insets for devices.
@@ -1239,6 +1379,19 @@ iOS: Values are expressed in physical pixels and exclude status bar.
 | **`top`**         | <code>number</code> | Orientation-aware notch/camera cutout inset (excluding status bar). In portrait mode: returns top inset (notch at top). In landscape mode: returns left inset (notch at side). Android: Value in dp, iOS: Value in pixels (status bar excluded). |
 
 
+#### FaceDetectionOptions
+
+Options for configuring face detection behavior.
+
+| Prop                  | Type                              | Description                                                                                                                                                                          | Default             |
+| --------------------- | --------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------- |
+| **`performanceMode`** | <code>'fast' \| 'accurate'</code> | Performance mode for face detection. - 'fast': Faster detection with slightly lower accuracy (recommended for real-time filters) - 'accurate': Slower detection with higher accuracy | <code>'fast'</code> |
+| **`trackingEnabled`** | <code>boolean</code>              | Enable face tracking across frames. When enabled, each face gets a unique tracking ID that persists across frames.                                                                   | <code>true</code>   |
+| **`detectLandmarks`** | <code>boolean</code>              | Detect facial landmarks (eyes, nose, mouth, etc.).                                                                                                                                   | <code>true</code>   |
+| **`maxFaces`**        | <code>number</code>               | Maximum number of faces to detect. Lower values improve performance.                                                                                                                 | <code>3</code>      |
+| **`minFaceSize`**     | <code>number</code>               | Minimum face size as a ratio of the frame width (0.0 - 1.0). Smaller faces than this will not be detected.                                                                           | <code>0.15</code>   |
+
+
 ### Type Aliases
 
 

android/build.gradle

@@ -36,6 +36,10 @@ android {
     lintOptions {
         abortOnError = false
     }
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_17
+        targetCompatibility JavaVersion.VERSION_17
+    }
 }
 
 repositories {
@@ -62,6 +66,9 @@ dependencies {
     implementation "androidx.camera:camera-view:${camerax_version}"
     implementation "androidx.camera:camera-extensions:${camerax_version}"
 
+    // ML Kit Face Detection
+    implementation 'com.google.mlkit:face-detection:16.1.7'
+
     testImplementation "junit:junit:$junitVersion"
     androidTestImplementation "androidx.test.ext:junit:$androidxJunitVersion"
     androidTestImplementation "androidx.test.espresso:espresso-core:$androidxEspressoCoreVersion"

android/src/main/java/app/capgo/capacitor/camera/preview/CameraPreview.java

@@ -68,6 +68,12 @@ public class CameraPreview extends Plugin implements CameraXView.CameraXViewList
     @Override
     protected void handleOnPause() {
         super.handleOnPause();
+
+        // Pause face detection when app goes to background
+        if (faceDetectionManager != null) {
+            faceDetectionManager.onAppBackground();
+        }
+
         if (cameraXView != null && cameraXView.isRunning()) {
             // Store the current configuration before stopping
             lastSessionConfig = cameraXView.getSessionConfig();
@@ -78,6 +84,12 @@ protected void handleOnPause() {
     @Override
     protected void handleOnResume() {
         super.handleOnResume();
+
+        // Resume face detection when app comes to foreground
+        if (faceDetectionManager != null) {
+            faceDetectionManager.onAppForeground();
+        }
+
         if (lastSessionConfig != null) {
             // Recreate camera with last known configuration
             if (cameraXView == null) {
@@ -1976,4 +1988,98 @@ public void getPluginVersion(final PluginCall call) {
             call.reject("Could not get plugin version", e);
         }
     }
+
+    // ========================================
+    // Face Detection Methods
+    // ========================================
+
+    private FaceDetectionManager faceDetectionManager;
+
+    @PluginMethod
+    public void startFaceDetection(final PluginCall call) {
+        if (cameraXView == null || !cameraXView.isRunning()) {
+            call.reject("Camera is not running. Start the camera preview first.");
+            return;
+        }
+
+        try {
+            // Get options from call
+            JSObject options = new JSObject();
+            if (call.getString("performanceMode") != null) {
+                options.put("performanceMode", call.getString("performanceMode"));
+            }
+            if (call.getBoolean("trackingEnabled") != null) {
+                options.put("trackingEnabled", call.getBoolean("trackingEnabled"));
+            }
+            if (call.getBoolean("detectLandmarks") != null) {
+                options.put("detectLandmarks", call.getBoolean("detectLandmarks"));
+            }
+            if (call.getBoolean("detectClassifications") != null) {
+                options.put("detectClassifications", call.getBoolean("detectClassifications"));
+            }
+            if (call.getInt("maxFaces") != null) {
+                options.put("maxFaces", call.getInt("maxFaces"));
+            }
+            if (call.getDouble("minFaceSize") != null) {
+                options.put("minFaceSize", call.getDouble("minFaceSize"));
+            }
+
+            // Create face detection manager if needed
+            if (faceDetectionManager == null) {
+                faceDetectionManager = new FaceDetectionManager();
+            }
+
+            // Start detection
+            faceDetectionManager.startDetection(
+                options,
+                new FaceDetectionManager.FaceDetectionListener() {
+                    @Override
+                    public void onFaceDetectionResult(org.json.JSONObject result) {
+                        // Convert org.json.JSONObject to JSObject
+                        try {
+                            JSObject jsResult = JSObject.fromJSONObject(result);
+                            notifyListeners("faceDetection", jsResult);
+                        } catch (Exception e) {
+                            Log.e(TAG, "Failed to convert face detection result", e);
+                        }
+                    }
+
+                    @Override
+                    public void onFaceDetectionError(String error) {
+                        Log.e(TAG, "Face detection error: " + error);
+                    }
+                }
+            );
+
+            // Enable face detection in camera view
+            cameraXView.enableFaceDetection(faceDetectionManager);
+
+            call.resolve();
+        } catch (Exception e) {
+            call.reject("Failed to start face detection: " + e.getMessage());
+        }
+    }
+
+    @PluginMethod
+    public void stopFaceDetection(final PluginCall call) {
+        try {
+            if (faceDetectionManager != null) {
+                faceDetectionManager.stopDetection();
+            }
+            if (cameraXView != null) {
+                cameraXView.disableFaceDetection();
+            }
+            call.resolve();
+        } catch (Exception e) {
+            call.reject("Failed to stop face detection: " + e.getMessage());
+        }
+    }
+
+    @PluginMethod
+    public void isFaceDetectionRunning(final PluginCall call) {
+        boolean isDetecting = faceDetectionManager != null && faceDetectionManager.isRunning();
+        JSObject ret = new JSObject();
+        ret.put("isDetecting", isDetecting);
+        call.resolve(ret);
+    }
 }