FCE-321: iOS camera in background; FCE-1727: Add PiP support to mobile-sdk (#505)

## Description

- Picture in Picture. Displayes main and secondary tracks if available,
secondary track is determined by VAD.
- On iOS support for background camera while in PiP

This is still a bit of an ongoing effort. UI and track switch should be
improved in next PRs. This PR introduces the main concept which works.

## Motivation and Context

- Customers need PiP when dealing with calls in background

## Documentation impact

- [x] Documentation update required
- [ ] Documentation updated [in another PR](_)
- [ ] No documentation update required

## Types of changes

- [ ] Bug fix (non-breaking change which fixes an issue)
- [x] New feature (non-breaking change which adds functionality)
- [ ] Breaking change (fix or feature that would cause existing
functionality to
      not work as expected)

---------

Co-authored-by: Anna Olak <anna.olak@swmansion.com>

Author: MiloszFilimowski

internal/fishjam-chat/app.json

@@ -62,11 +62,13 @@
         "@fishjam-cloud/react-native-client",
         {
           "android": {
-            "enableForegroundService": true
+            "enableForegroundService": true,
+            "supportsPictureInPicture": true
           },
           "ios": {
             "enableScreensharing": true,
-            "enableVoIPBackgroundMode": true
+            "enableVoIPBackgroundMode": true,
+            "supportsPictureInPicture": true
           }
         }
       ],

internal/fishjam-chat/components/VideosGrid/index.tsx

@@ -6,6 +6,7 @@ import { PeerMetadata } from '../../types/metadata';
 import NoCameraView from '../NoCameraView';
 import { GridTrack, GridTrackItem } from './GridTrackItem';
 import { parsePeersToTracks } from './parsePeersToTracks';
+import { PipContainerView } from '@fishjam-cloud/react-native-client';
 
 const ListFooterComponent = () => <View style={{ height: 60 }} />;
 
@@ -39,14 +40,19 @@ export default function VideosGrid({
   );
 
   return (
-    <FlatList<GridTrack>
-      data={videoTracks}
-      keyExtractor={keyExtractor}
-      renderItem={renderItem}
-      contentContainerStyle={styles.contentContainerStyle}
-      ListFooterComponent={ListFooterComponent}
-      ListEmptyComponent={ListEmptyComponent}
-    />
+    <PipContainerView
+      allowsCameraInBackground
+      startAutomatically
+      stopAutomatically>
+      <FlatList<GridTrack>
+        data={videoTracks}
+        keyExtractor={keyExtractor}
+        renderItem={renderItem}
+        contentContainerStyle={styles.contentContainerStyle}
+        ListFooterComponent={ListFooterComponent}
+        ListEmptyComponent={ListEmptyComponent}
+      />
+    </PipContainerView>
   );
 }
 

packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/ReconnectionManager.kt

@@ -3,6 +3,7 @@ package com.fishjamcloud.client
 import com.fishjamcloud.client.models.ReconnectConfig
 import timber.log.Timber
 import java.util.Timer
+import java.util.concurrent.CopyOnWriteArrayList
 import kotlin.concurrent.schedule
 
 interface ReconnectionManagerListener {
@@ -31,7 +32,7 @@ internal class ReconnectionManager(
   private var reconnectConfig: ReconnectConfig = ReconnectConfig(),
   private val connect: () -> Unit
 ) {
-  private val listeners = mutableListOf<ReconnectionManagerListener>()
+  private val listeners = CopyOnWriteArrayList<ReconnectionManagerListener>()
   private var reconnectAttempts = 0
   private var reconnectionStatus = ReconnectionStatus.Idle
 

packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/PeerConnectionManager.kt

@@ -24,11 +24,12 @@ import kotlinx.coroutines.sync.withLock
 import org.webrtc.*
 import timber.log.Timber
 import java.util.*
+import java.util.concurrent.CopyOnWriteArrayList
 
 internal class PeerConnectionManager(
   private val peerConnectionFactory: PeerConnectionFactoryWrapper
 ) : PeerConnection.Observer {
-  private val listeners = mutableListOf<PeerConnectionListener>()
+  private val listeners = CopyOnWriteArrayList<PeerConnectionListener>()
 
   fun addListener(listener: PeerConnectionListener) {
     listeners.add(listener)

packages/android-client/FishjamClient/src/main/java/com/fishjamcloud/client/webrtc/RTCEngineCommunication.kt

@@ -7,9 +7,10 @@ import com.fishjamcloud.client.utils.gson
 import com.fishjamcloud.client.utils.serializeToMap
 import fishjam.media_events.Shared
 import timber.log.Timber
+import java.util.concurrent.CopyOnWriteArrayList
 
 internal class RTCEngineCommunication {
-  private val listeners = mutableListOf<RTCEngineListener>()
+  private val listeners = CopyOnWriteArrayList<RTCEngineListener>()
 
   fun addListener(listener: RTCEngineListener) {
     listeners.add(listener)

packages/ios-client/Sources/FishjamClient/FishjamClientInternal.swift

@@ -896,7 +896,7 @@ extension FishjamClientInternal: RTCEngineListener {
     }
 
     func onTrackUpdated(endpointId: String, trackId: String, metadata: Metadata) {
-        guard let track = getTrack(trackId: trackId) else {
+        guard let track = getTrackWithRtcEngineId(trackId: trackId) else {
             sdkLogger.error("Failed to process TrackUpdated event: Track context not found: \(trackId)")
             return
         }

packages/ios-client/Sources/FishjamClient/media/Capturers/CameraCapturer.swift

@@ -8,6 +8,7 @@ public protocol CameraCapturerDeviceChangedListener: AnyObject {
 class CameraCapturer: VideoCapturer {
     private let videoParameters: VideoParameters
     private let capturer: RTCCameraVideoCapturer
+    private let captureSession: AVCaptureSession
     internal var isFront: Bool = true
     private(set) var device: AVCaptureDevice? {
         didSet {
@@ -19,7 +20,8 @@ class CameraCapturer: VideoCapturer {
 
     init(videoParameters: VideoParameters, delegate: RTCVideoCapturerDelegate, deviceId: String? = nil) {
         self.videoParameters = videoParameters
-        self.capturer = RTCCameraVideoCapturer(delegate: delegate, captureSession: AVCaptureSession())
+        self.captureSession = AVCaptureSession()
+        self.capturer = RTCCameraVideoCapturer(delegate: delegate, captureSession: captureSession)
         let devices = RTCCameraVideoCapturer.captureDevices()
 
         if let newDevice = devices.first(where: { $0.uniqueID == deviceId }) {
@@ -113,4 +115,29 @@ class CameraCapturer: VideoCapturer {
             }
         )
     }
+
+    public func isMultitaskingCameraAccessSupported() -> Bool {
+        if #available(iOS 16.0, *) {
+            return captureSession.isMultitaskingCameraAccessSupported
+        }
+        return false
+    }
+
+    public func setMultitaskingCameraAccessEnabled(_ enabled: Bool) -> Bool {
+        if #available(iOS 16.0, *) {
+            guard captureSession.isMultitaskingCameraAccessSupported else {
+                return false
+            }
+            captureSession.isMultitaskingCameraAccessEnabled = enabled
+            return true
+        }
+        return false
+    }
+
+    public func isMultitaskingCameraAccessEnabled() -> Bool {
+        if #available(iOS 16.0, *) {
+            return captureSession.isMultitaskingCameraAccessEnabled
+        }
+        return false
+    }
 }

packages/ios-client/Sources/FishjamClient/media/Tracks/LocalCameraTrack.swift

@@ -72,4 +72,16 @@ public class LocalCameraTrack: LocalVideoTrack, LocalTrack {
     public static func getCaptureDevices() -> [AVCaptureDevice] {
         return RTCCameraVideoCapturer.captureDevices()
     }
+
+    public func isMultitaskingCameraAccessSupported() -> Bool {
+        return capturer.isMultitaskingCameraAccessSupported()
+    }
+
+    public func setMultitaskingCameraAccessEnabled(_ enabled: Bool) -> Bool {
+        return capturer.setMultitaskingCameraAccessEnabled(enabled)
+    }
+
+    public func isMultitaskingCameraAccessEnabled() -> Bool {
+        return capturer.isMultitaskingCameraAccessEnabled()
+    }
 }

packages/ios-client/Sources/FishjamClient/media/Tracks/Track.swift

@@ -1,7 +1,7 @@
 import WebRTC
 
 open class Track: Equatable {
-    internal var mediaTrack: RTCMediaStreamTrack?
+    public internal(set) var mediaTrack: RTCMediaStreamTrack?
     internal var sendEncodings: [RTCRtpEncodingParameters] = []
     let endpointId: String
     internal var rtcEngineId: String?