Background blurring 👤✨

.gitignore

@@ -88,3 +88,4 @@ freeline_project_description.json
 **/__pycache__/
 
 /gradle/verification-keyring.gpg
+/app/src/main/assets/selfie_segmenter.tflite

app/build.gradle

@@ -16,6 +16,7 @@ plugins {
     id "org.jetbrains.kotlin.plugin.compose" version "2.2.20"
     id "org.jetbrains.kotlin.kapt"
     id 'com.google.devtools.ksp' version '2.2.20-2.0.3'
+    id "de.undercouch.download" version "5.6.0" // Download task and extension for retrieving files during a build
 }
 
 apply plugin: 'com.android.application'
@@ -184,6 +185,9 @@ configurations.configureEach {
     exclude group: 'org.jetbrains', module: 'annotations-java5' // via prism4j, already using annotations explicitly
 }
 
+project.ext.ASSET_DIR = projectDir.toString() + '/src/main/assets'
+apply from: 'download_model.gradle'
+
 dependencies {
     implementation "androidx.room:room-testing-android:${roomVersion}"
     implementation 'androidx.compose.foundation:foundation-layout:1.9.1'
@@ -362,6 +366,10 @@ dependencies {
     testImplementation("com.squareup.okhttp3:mockwebserver:$okhttpVersion")
     testImplementation("com.google.dagger:hilt-android-testing:2.57.1")
     testImplementation("org.robolectric:robolectric:4.16")
+
+    // Computer Vision - for background effects during video calls
+    implementation 'com.google.mediapipe:tasks-vision:0.10.26'
+    implementation 'org.opencv:opencv:4.12.0'
 }
 
 tasks.register('installGitHooks', Copy) {

app/download_model.gradle

@@ -0,0 +1,15 @@
+/*
+ * Nextcloud Talk - Android Client
+ *
+ * SPDX-FileCopyrightText: 2025 Julius Linus <[email protected]>
+ * SPDX-License-Identifier: GPL-3.0-or-later
+ */
+
+
+task downloadSelfieSegmenterModelFile(type: Download) {
+    src 'https://storage.googleapis.com/mediapipe-models/image_segmenter/selfie_segmenter/float16/1/selfie_segmenter.tflite'
+    dest project.ext.ASSET_DIR + '/selfie_segmenter.tflite'
+    overwrite false
+}
+
+preBuild.dependsOn downloadSelfieSegmenterModelFile

app/src/main/java/com/nextcloud/talk/activities/CallActivity.kt

@@ -75,6 +75,9 @@ import com.nextcloud.talk.call.MessageSenderNoMcu
 import com.nextcloud.talk.call.MutableLocalCallParticipantModel
 import com.nextcloud.talk.call.ReactionAnimator
 import com.nextcloud.talk.call.components.ParticipantGrid
+import com.nextcloud.talk.camera.BackgroundBlurFrameProcessor
+import com.nextcloud.talk.camera.BlurBackgroundViewModel
+import com.nextcloud.talk.camera.BlurBackgroundViewModel.BackgroundBlurOn
 import com.nextcloud.talk.chat.ChatActivity
 import com.nextcloud.talk.data.user.model.User
 import com.nextcloud.talk.databinding.CallActivityBinding
@@ -187,7 +190,7 @@ import kotlin.math.abs
 import kotlin.math.roundToInt
 
 @AutoInjector(NextcloudTalkApplication::class)
-@Suppress("TooManyFunctions")
+@Suppress("TooManyFunctions", "ReturnCount", "LargeClass")
 class CallActivity : CallBaseActivity() {
     @JvmField
     @Inject
@@ -210,6 +213,7 @@ class CallActivity : CallBaseActivity() {
     var audioManager: WebRtcAudioManager? = null
     var callRecordingViewModel: CallRecordingViewModel? = null
     var raiseHandViewModel: RaiseHandViewModel? = null
+    val blurBackgroundViewModel: BlurBackgroundViewModel = BlurBackgroundViewModel()
     private var mReceiver: BroadcastReceiver? = null
     private var peerConnectionFactory: PeerConnectionFactory? = null
     private var audioConstraints: MediaConstraints? = null
@@ -234,7 +238,7 @@ class CallActivity : CallBaseActivity() {
     private val peerConnectionWrapperList: MutableList<PeerConnectionWrapper> = ArrayList()
     private var videoOn = false
     private var microphoneOn = false
-    private var isVoiceOnlyCall = false
+    var isVoiceOnlyCall = false
     private var isCallWithoutNotification = false
     private var isIncomingCallFromNotification = false
     private val callControlHandler = Handler()
@@ -395,6 +399,8 @@ class CallActivity : CallBaseActivity() {
 
         initRaiseHandViewModel()
         initCallRecordingViewModel(intent.extras!!.getInt(KEY_RECORDING_STATE))
+        initBackgroundBlurViewModel()
+
         initClickListeners(isModerator, isOneToOneConversation)
         binding!!.microphoneButton.setOnTouchListener(MicrophoneButtonTouchListener())
         pulseAnimation = PulseAnimation.create().with(binding!!.microphoneButton)
@@ -487,6 +493,26 @@ class CallActivity : CallBaseActivity() {
         }
     }
 
+    private fun initBackgroundBlurViewModel() {
+        blurBackgroundViewModel.viewState.observe(this) { state ->
+            val frontFacing = isCameraFrontFacing(cameraEnumerator)
+            if (frontFacing == null) {
+                Log.e(TAG, "Camera not found")
+                return@observe
+            }
+
+            val isOn = state == BackgroundBlurOn
+
+            val processor = if (isOn) {
+                BackgroundBlurFrameProcessor(context, frontFacing)
+            } else {
+                null
+            }
+
+            videoSource?.setVideoProcessor(processor)
+        }
+    }
+
     private fun processExtras(extras: Bundle) {
         roomId = extras.getString(KEY_ROOM_ID, "")
         roomToken = extras.getString(KEY_ROOM_TOKEN, "")
@@ -1106,6 +1132,7 @@ class CallActivity : CallBaseActivity() {
                 rootEglBase!!.eglBaseContext
             )
             videoSource = peerConnectionFactory!!.createVideoSource(false)
+
             videoCapturer!!.initialize(surfaceTextureHelper, applicationContext, videoSource!!.capturerObserver)
         }
         localVideoTrack = peerConnectionFactory!!.createVideoTrack("NCv0", videoSource)
@@ -1186,6 +1213,30 @@ class CallActivity : CallBaseActivity() {
         return null
     }
 
+    private fun isCameraFrontFacing(enumerator: CameraEnumerator?): Boolean? {
+        if (enumerator == null) {
+            return false
+        }
+
+        val deviceNames = enumerator.deviceNames
+
+        // First, try to find front facing camera
+        for (deviceName in deviceNames) {
+            if (enumerator.isFrontFacing(deviceName)) {
+                return true
+            }
+        }
+
+        // Front facing camera not found, try something else
+        for (deviceName in deviceNames) {
+            if (!enumerator.isFrontFacing(deviceName)) {
+                return false
+            }
+        }
+
+        return null
+    }
+
     fun onMicrophoneClick() {
         if (!canPublishAudioStream) {
             microphoneOn = false
@@ -1272,6 +1323,7 @@ class CallActivity : CallBaseActivity() {
             } else {
                 binding!!.cameraButton.setImageResource(R.drawable.ic_videocam_off_white_24px)
                 binding!!.switchSelfVideoButton.visibility = View.GONE
+                blurBackgroundViewModel.turnOffBlur()
             }
             toggleMedia(videoOn, true)
         } else if (shouldShowRequestPermissionRationale(Manifest.permission.CAMERA)) {
@@ -1351,6 +1403,10 @@ class CallActivity : CallBaseActivity() {
         raiseHandViewModel!!.clickHandButton()
     }
 
+    fun toggleBackgroundBlur() {
+        blurBackgroundViewModel.toggleBackgroundBlur()
+    }
+
     private fun animateCallControls(show: Boolean, startDelay: Long) {
         if (isVoiceOnlyCall) {
             if (spotlightView != null && spotlightView!!.visibility != View.GONE) {