Live source thumbnails and per-person Swap assignment

work/android/app/src/main/cpp/ffjni.cpp

@@ -265,6 +265,12 @@ Java_com_facefusion_mobile_NativePipe_setFaceAssignEnabled(JNIEnv*, jclass, jboo
   if (g_pipe) g_pipe->setFaceAssignEnabled(g_assignEnabled);
 }
 
+JNIEXPORT void JNICALL
+Java_com_facefusion_mobile_NativePipe_setSelectedFaceDisabled(JNIEnv*, jclass,
+                                                               jboolean disabled) {
+  if (g_pipe) g_pipe->setSelectedFaceDisabled(disabled == JNI_TRUE);
+}
+
 // The SELECTED person (assign mode): the last one tapped, who follows the source chip
 // until an empty tap deselects them. FIVE floats -- x0, y0, x1, y1 and the person's
 // CURRENT source -- in DISPLAY bitmap coordinates, so the UI can draw a persistent
@@ -284,6 +290,44 @@ Java_com_facefusion_mobile_NativePipe_takeSelectionBox(JNIEnv* env, jclass) {
   return out;
 }
 
+JNIEXPORT jfloatArray JNICALL
+Java_com_facefusion_mobile_NativePipe_assignFaceAt(JNIEnv* env, jclass,
+                                                   jbyteArray jBgr, jint w, jint h,
+                                                   jfloat x, jfloat y, jint source,
+                                                   jboolean noFace) {
+  if (!g_pipe) { g_err = "pipeline not initialised"; return env->NewFloatArray(0); }
+  ffcv::Image img(w, h, 3);
+  if (!jBgr || (size_t)env->GetArrayLength(jBgr) != img.data.size()) {
+    g_err = "assignFaceAt: frame is not w*h*3 bytes";
+    return env->NewFloatArray(0);
+  }
+  env->GetByteArrayRegion(jBgr, 0, (jsize)img.data.size(), (jbyte*)img.data.data());
+  float box[4] = {0, 0, 0, 0};
+  float embedding[512] = {0};
+  if (!g_pipe->setFaceSourceAt(img, x, y, (int)source, noFace == JNI_TRUE, box, embedding)) {
+    g_err = g_pipe->error();
+    return env->NewFloatArray(0);
+  }
+  // Return the embedding for both choices: the no-face flag is stored natively, while
+  // Kotlin needs the identity to restore this assignment after a pipeline reload.
+  jfloatArray out = env->NewFloatArray(512);
+  if (out) env->SetFloatArrayRegion(out, 0, 512, embedding);
+  return out;
+}
+
+JNIEXPORT jboolean JNICALL
+Java_com_facefusion_mobile_NativePipe_addFaceAssignmentEmbedding(JNIEnv* env, jclass,
+                                                                jfloatArray jEmbedding,
+                                                                jint source) {
+  if (!g_pipe || !jEmbedding || env->GetArrayLength(jEmbedding) != 512) {
+    g_err = "invalid face assignment embedding";
+    return JNI_FALSE;
+  }
+  float embedding[512] = {0};
+  env->GetFloatArrayRegion(jEmbedding, 0, 512, embedding);
+  return g_pipe->addFaceAssignmentEmbedding(embedding, (int)source) ? JNI_TRUE : JNI_FALSE;
+}
+
 JNIEXPORT void JNICALL
 Java_com_facefusion_mobile_NativePipe_clearFaceSourceAssignments(JNIEnv*, jclass) {
   if (g_pipe) g_pipe->clearFaceSourceAssignments();
@@ -1066,7 +1110,9 @@ Java_com_facefusion_mobile_NativePipe_liveFrame(JNIEnv* env, jclass,
   // geometry only this function knows. consumed is set whether or not the tap hit a
   // face: a miss must be reported, not left hanging.
   const bool tapPending = g_assignReq.pending;
+  int requestedSource = -1;
   if (tapPending) {
+    requestedSource = g_assignReq.source;
     g_assignReq.pending = false;
     g_assignResult.consumed = true;
     g_assignResult.have = false;
@@ -1082,7 +1128,7 @@ Java_com_facefusion_mobile_NativePipe_liveFrame(JNIEnv* env, jclass,
       faces,
       tapPending ? g_assignReq.x * (float)w / (float)dw : 0.f,
       tapPending ? g_assignReq.y * (float)h / (float)dh : 0.f,
-      tapPending ? (int)g_assignReq.source : -1,
+      tapPending ? requestedSource : -1,
       tapBox);
   if (tapPending && tapped) {
     g_assignResult.have = true;
@@ -1092,7 +1138,8 @@ Java_com_facefusion_mobile_NativePipe_liveFrame(JNIEnv* env, jclass,
     g_assignResult.box[1] = tapBox[1] * (float)dh / (float)h;
     g_assignResult.box[2] = tapBox[2] * (float)dw / (float)w;
     g_assignResult.box[3] = tapBox[3] * (float)dh / (float)h;
-    g_assignResult.source = (int)g_assignReq.source;
+    // -2 is the UI's No face brush; expose it as a negative source to the overlay.
+    g_assignResult.source = requestedSource == -2 ? -1 : requestedSource;
   }
 
   if (!faces.empty()) {

work/android/app/src/main/cpp/ffpipe.cpp

@@ -1034,9 +1034,9 @@ void Pipeline::setActiveSource(int index) {
 }
 
 bool Pipeline::setFaceSourceAt(const ffcv::Image& frame, float x, float y, int sourceIndex,
-                               bool disabled, float* outBox) {
+                               bool disabled, float* outBox, float* outEmbedding) {
   err_.clear();
-  if (sourceIndex < 0 || sourceIndex >= (int)p_->sourceSlots.size()) {
+  if ((sourceIndex < 0 && !disabled) || sourceIndex >= (int)p_->sourceSlots.size()) {
     err_ = "source index out of range"; return false;
   }
   // noTrack, like every source-image analysis: this records an identity, and an identity
@@ -1049,12 +1049,40 @@ bool Pipeline::setFaceSourceAt(const ffcv::Image& frame, float x, float y, int s
   }
   if (!chosen) { err_ = "no face at selected point"; return false; }
   if (outBox) std::memcpy(outBox, chosen->box, sizeof(float) * 4);
+  if (outEmbedding) std::memcpy(outEmbedding, chosen->embeddingNorm, sizeof(float) * 512);
+  const float kSamePerson = 0.15f;
+  p_->faceAssignments.erase(
+      std::remove_if(p_->faceAssignments.begin(), p_->faceAssignments.end(),
+          [&](const Pipeline::Impl::FaceAssignment& existing) {
+            return faceDistance(chosen->embeddingNorm, existing.embedding.data()) < kSamePerson;
+          }),
+      p_->faceAssignments.end());
   Pipeline::Impl::FaceAssignment a{}; std::memcpy(a.embedding.data(), chosen->embeddingNorm, sizeof(float) * 512);
   a.source = sourceIndex; a.disabled = disabled;
   p_->faceAssignments.push_back(a);
   return true;
 }
 
+bool Pipeline::addFaceAssignmentEmbedding(const float* embedding, int sourceIndex) {
+  if (!p_ || !embedding || sourceIndex < -1 || sourceIndex >= (int)p_->sourceSlots.size()) {
+    err_ = "invalid face assignment";
+    return false;
+  }
+  const float kSamePerson = 0.15f;
+  p_->faceAssignments.erase(
+      std::remove_if(p_->faceAssignments.begin(), p_->faceAssignments.end(),
+          [&](const Pipeline::Impl::FaceAssignment& existing) {
+            return faceDistance(embedding, existing.embedding.data()) < kSamePerson;
+          }),
+      p_->faceAssignments.end());
+  Pipeline::Impl::FaceAssignment a{};
+  std::memcpy(a.embedding.data(), embedding, sizeof(float) * 512);
+  a.source = sourceIndex;
+  a.disabled = sourceIndex < 0;
+  p_->faceAssignments.push_back(a);
+  return true;
+}
+
 void Pipeline::clearFaceSourceAssignments() {
   p_->faceAssignments.clear();
   // The people being tracked are only "who has which source" because of the
@@ -1063,6 +1091,29 @@ void Pipeline::clearFaceSourceAssignments() {
   p_->tracked.clear(); p_->frameSources.clear(); p_->selectedId = -1;
 }
 
+void Pipeline::setSelectedFaceDisabled(bool disabled) {
+  if (!p_ || !p_->assignEnabled || p_->selectedId < 0) return;
+  for (auto& t : p_->tracked) {
+    if (t.id != p_->selectedId) continue;
+    const float kSamePerson = 0.15f;
+    p_->faceAssignments.erase(
+        std::remove_if(p_->faceAssignments.begin(), p_->faceAssignments.end(),
+            [&](const Pipeline::Impl::FaceAssignment& a) {
+              return faceDistance(t.embeddingNorm, a.embedding.data()) < kSamePerson;
+            }),
+        p_->faceAssignments.end());
+    t.source = disabled ? -1 : p_->activeSource;
+    t.pinned = true;
+    t.missed = 0;
+    Pipeline::Impl::FaceAssignment a{};
+    std::memcpy(a.embedding.data(), t.embeddingNorm, sizeof(float) * 512);
+    a.source = disabled ? -1 : p_->activeSource;
+    a.disabled = disabled;
+    p_->faceAssignments.push_back(a);
+    return;
+  }
+}
+
 void Pipeline::setFaceAssignEnabled(bool enabled) {
   p_->assignEnabled = enabled;
   // Enabling drops whatever the detector tracker was holding: assign mode analyses with
@@ -1149,6 +1200,10 @@ bool Pipeline::updateLiveTracking(const std::vector<Face>& faces,
     return false;
   }
   err_.clear();
+  // -2 is the Live UI's No face brush. Internally use slot 0 for validation, then
+  // preserve the disabled flag when pinning the tracked person.
+  const bool tapDisabled = tapSource == -2;
+  if (tapDisabled) tapSource = 0;
   const int kMaxTracked = 16;
   const float kMatchDist = 1.3f;     // centre distance, in box-dimension units
   const float kTapDist = 1.0f;       // how far a tap may be from a box centre
@@ -1291,7 +1346,7 @@ bool Pipeline::updateLiveTracking(const std::vector<Face>& faces,
       else if (d < secondD) secondD = d;
     }
     if (hit && secondD - bestD > kAdoptMargin &&
-        hit->source >= 0 && hit->source < (int)p_->sourceSlots.size()) {
+        (hit->disabled || (hit->source >= 0 && hit->source < (int)p_->sourceSlots.size()))) {
       t.source = hit->source; t.pinned = true;
     }
     t.id = p_->nextTrackedId++;
@@ -1304,7 +1359,7 @@ bool Pipeline::updateLiveTracking(const std::vector<Face>& faces,
   // only found a current face pins the entry that face has (creating it if the tap
   // preceded the association). The pin lands in the table the swap on THIS frame reads,
   // so the new source applies immediately.
-  if (tapSource >= 0 && (tappedEntry >= 0 || tappedFace >= 0)) {
+  if ((tapDisabled || tapSource >= 0) && (tappedEntry >= 0 || tappedFace >= 0)) {
     int idx = tappedEntry;
     if (idx < 0) {
       idx = faceTrack[(size_t)tappedFace];
@@ -1317,7 +1372,7 @@ bool Pipeline::updateLiveTracking(const std::vector<Face>& faces,
       }
     }
     Pipeline::Impl::TrackedFace& t = trk[(size_t)idx];
-    t.source = tapSource; t.pinned = true; t.missed = 0;
+    t.source = tapDisabled ? -1 : tapSource; t.pinned = true; t.missed = 0;
     // The tapped person becomes the SELECTED one: they follow the chip until an empty
     // tap deselects them (see the miss branch above).
     p_->selectedId = t.id;
@@ -1348,7 +1403,8 @@ bool Pipeline::updateLiveTracking(const std::vector<Face>& faces,
         p_->faceAssignments.end());
     Pipeline::Impl::FaceAssignment a{};
     std::memcpy(a.embedding.data(), t.embeddingNorm, sizeof(float) * 512);
-    a.source = tapSource;
+    a.source = tapDisabled ? -1 : tapSource;
+    a.disabled = tapDisabled;
     p_->faceAssignments.push_back(a);
     if (outTapBox) std::memcpy(outTapBox, t.box, sizeof(float) * 4);
   }
@@ -1590,6 +1646,10 @@ bool Pipeline::swapAll(ffcv::Image& frame, const std::vector<Face>& faces) {
   for (size_t fi = 0; fi < faces.size(); ++fi) {
     const Face& f = faces[fi];
     if (only && &f != only) continue;
+    // A negative per-person source is the explicit "No face" assignment. Leave the
+    // target pixels untouched, including when the enhancer is enabled later.
+    if (p_->assignEnabled && p_->frameSources.size() == faces.size() &&
+        p_->frameSources[fi] < 0) continue;
     // face_selector.py:find_match_faces -- ALL faces near the reference, not the nearest
     // one. Upstream returns a list, and a clip where the same person is detected twice
     // (a reflection, a poster) should swap both rather than pick between them.
@@ -1610,6 +1670,28 @@ bool Pipeline::swapAll(ffcv::Image& frame, const std::vector<Face>& faces) {
     // vector, but a stale index must still fall back to slot 0 rather than read past
     // it.
     int six = p_->activeSource;
+    // Swap's Assign per person mode uses the target identity assignments on every frame.
+    // Live still fills frameSources for its frame-to-frame tracker, which takes priority.
+    if (p_->assignEnabled && p_->frameSources.size() != faces.size()) {
+      const Pipeline::Impl::FaceAssignment* bestAssignment = nullptr;
+      float bestDistance = cfg.referenceDistance;
+      for (const auto& assignment : p_->faceAssignments) {
+        // A disabled assignment is still a real assignment: it means this matched
+        // person must remain untouched, not that the entry should be ignored.
+        if ((!assignment.disabled && assignment.source < 0) ||
+            (!assignment.disabled && assignment.source >= (int)p_->sourceSlots.size()))
+          continue;
+        const float distance = faceDistance(f.embeddingNorm, assignment.embedding.data());
+        if (distance < bestDistance) {
+          bestDistance = distance;
+          bestAssignment = &assignment;
+        }
+      }
+      if (bestAssignment) {
+        if (bestAssignment->disabled) continue;
+        six = bestAssignment->source;
+      }
+    }
     // PER-PERSON ASSIGNMENT (Live, switch ON): the per-face table updateLiveTracking
     // built this frame overrides the slot. A PINNED face keeps ITS source -- sticky,
     // decided once at the tap and never re-scored against per-frame embedding noise,
@@ -1619,6 +1701,7 @@ bool Pipeline::swapAll(ffcv::Image& frame, const std::vector<Face>& faces) {
     // is byte-for-byte the active-slot path below.
     if (p_->assignEnabled && p_->frameSources.size() == faces.size())
       six = p_->frameSources[fi];
+    if (six < 0) continue;
     const float* srcEmbNorm = p_->srcEmbeddingNorm;
     if (!p_->sourceSlots.empty() && six >= 0 && six < (int)p_->sourceSlots.size())
       srcEmbNorm = p_->sourceSlots[(size_t)six].data();
@@ -1717,8 +1800,12 @@ bool Pipeline::enhance(ffcv::Image& frame, const std::vector<Face>& faces) {
   }
 
   const int ES = kEnhSize, EPB = CS / ES;
-  for (const Face& f : faces) {
+  for (size_t fi = 0; fi < faces.size(); ++fi) {
+    const Face& f = faces[fi];
     if (only && &f != only) continue;
+    // Keep an explicitly unmodified person untouched by the optional enhancer too.
+    if (p_->assignEnabled && p_->frameSources.size() == faces.size() &&
+        p_->frameSources[fi] < 0) continue;
 
     double t0 = nowMs();
     ffcv::Affine am = ffcv::umeyama(f.landmark5_68, tmpl, 5);

work/android/app/src/main/cpp/ffpipe.h

@@ -225,7 +225,8 @@ class Pipeline {
   void clearSourceSlots();
   void setActiveSource(int index);
   bool setFaceSourceAt(const ffcv::Image& frame, float x, float y, int sourceIndex,
-                       bool disabled, float* outBox);
+                       bool disabled, float* outBox, float* outEmbedding = nullptr);
+  bool addFaceAssignmentEmbedding(const float* embedding, int sourceIndex);
   void clearFaceSourceAssignments();
 
   /**
@@ -242,6 +243,9 @@ class Pipeline {
    */
   void setFaceAssignEnabled(bool enabled);
 
+  /** Immediately change the currently selected Live person to/from the original face. */
+  void setSelectedFaceDisabled(bool disabled);
+
   /**
    * Record that [f] -- a face of a LIVE frame, embedding included -- belongs to
    * [sourceIndex]. Called from liveFrame against the PRE-SWAP detections, so the

work/android/app/src/main/java/com/facefusion/mobile/LiveRecorder.kt

@@ -34,6 +34,8 @@ import java.io.File
 class LiveRecorder(
     private val out: File,
     private val microphone: LiveMicrophone? = null,
+    /** The recording must use the same horizontal orientation as the displayed Live feed. */
+    private val mirrorCamera: Boolean = false,
     private val onLog: (String) -> Unit = {},
 ) {
 
@@ -130,6 +132,11 @@ class LiveRecorder(
     /**
      * One swapped frame, BGR, [w]*[h]*3 bytes.
      *
+     * The preview mirror is a Compose display transform, so the bytes arriving here are
+     * still in camera order. Apply the same horizontal flip before converting to the
+     * encoder's YUV planes; this mutates the reusable recording buffer in place and avoids
+     * another full-frame allocation or a second pipeline pass.
+     *
      * Called on the analyzer thread, in line with the pump. Encoding a 720p frame costs a
      * few milliseconds against the pump's ~60, so it is not worth another thread and its
      * queue — and a queue would be the wrong answer anyway: dropping the recording behind
@@ -144,6 +151,7 @@ class LiveRecorder(
         // arrives here afterwards, and must do nothing at all. See [stopped].
         if (stopped || failed != null) return
         if (!ensure(w, h)) return
+        if (mirrorCamera) mirrorHorizontally(bgr, w, h)
         val enc = encoder ?: return
         val ew = w and 1.inv()
         val eh = h and 1.inv()
@@ -179,6 +187,23 @@ class LiveRecorder(
       }
     }
 
+    /** Flip packed BGR pixels in place so the encoder matches the mirrored display. */
+    private fun mirrorHorizontally(bgr: ByteArray, w: Int, h: Int) {
+        for (y in 0 until h) {
+            var left = 0
+            var right = w - 1
+            while (left < right) {
+                val li = (y * w + left) * 3
+                val ri = (y * w + right) * 3
+                val b = bgr[li]; bgr[li] = bgr[ri]; bgr[ri] = b
+                val g = bgr[li + 1]; bgr[li + 1] = bgr[ri + 1]; bgr[ri + 1] = g
+                val r = bgr[li + 2]; bgr[li + 2] = bgr[ri + 2]; bgr[ri + 2] = r
+                left++
+                right--
+            }
+        }
+    }
+
     /**
      * Finish the file.
      *