Handle the case where VLA-provided frame rate is less than the onstage preferred frame rate. (#2268)

Correctly calculate temporal layers' frame rates from VLA (presuming 2:1 ratios).

Author: JonathanLennox

jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/av1/Av1DDRtpLayerDesc.kt

@@ -79,8 +79,7 @@ class Av1DDRtpLayerDesc(
      * {@inheritDoc}
      */
     override fun toString(): String {
-        return "subjective_quality=" + index +
-            ",DT=" + dt
+        return "subjective_quality=$index,DT=$dt,height=$height,frameRate=$frameRate"
     }
 
     companion object {

jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/rtp/codec/vpx/VpxRtpLayerDesc.kt

@@ -118,7 +118,7 @@ constructor(
      * {@inheritDoc}
      */
     override fun toString(): String {
-        return "subjective_quality=$index,temporal_id=$tid,spatial_id=$sid,height=$height"
+        return "subjective_quality=$index,temporal_id=$tid,spatial_id=$sid,height=$height,frameRate=$frameRate"
     }
 
     /**

jitsi-media-transform/src/main/kotlin/org/jitsi/nlj/transform/node/incoming/VlaReaderNode.kt

@@ -76,22 +76,26 @@ class VlaReaderNode(
                 vla.forEachIndexed { streamIdx, stream ->
                     val rtpEncoding = sourceDesc?.rtpEncodings?.get(streamIdx)
                     stream.spatialLayers.forEach { spatialLayer ->
+                        val maxTl = spatialLayer.targetBitratesKbps.size - 1
+
                         spatialLayer.targetBitratesKbps.forEachIndexed { tlIdx, targetBitrateKbps ->
                             rtpEncoding?.layers?.find {
                                 // With VP8 simulcast all layers have sid -1
                                 (it.sid == spatialLayer.id || it.sid == -1) && it.tid == tlIdx
                             }?.let { layer ->
-                                logger.debug(
+                                logger.debug {
                                     "Setting target bitrate for rtpEncoding=$rtpEncoding layer=$layer to " +
                                         "${targetBitrateKbps.kbps} (res=${spatialLayer.res})"
-                                )
+                                }
                                 layer.targetBitrate = targetBitrateKbps.kbps
                                 spatialLayer.res?.let { res ->
                                     if (layer.height > 0 && layer.height != res.height) {
                                         logger.warn("Updating layer height from ${layer.height} to ${res.height}")
                                     }
                                     layer.height = res.height
-                                    layer.frameRate = res.maxFramerate.toDouble()
+                                    /* Presume 2:1 frame rate ratios for temporal layers */
+                                    val framerateFraction = 1.0 / (1 shl (maxTl - tlIdx))
+                                    layer.frameRate = res.maxFramerate.toDouble() * framerateFraction
                                 }
                             }
                         }

jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BandwidthAllocator.kt

@@ -174,7 +174,7 @@ internal class BandwidthAllocator<T : MediaSourceContainer>(
 
         logger.trace {
             "Allocating: sortedSources=${sortedSources.map { it.sourceName }}, " +
-                " effectiveConstraints=${newEffectiveConstraints.map { "${it.key.sourceName}=${it.value}" }}"
+                "effectiveConstraints=${newEffectiveConstraints.map { "${it.key.sourceName}=${it.value}" }}"
         }
 
         // Compute the bandwidth allocation.

jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/BitrateController.kt

@@ -26,6 +26,7 @@ import org.jitsi.utils.event.SyncEventEmitter
 import org.jitsi.utils.logging.DiagnosticContext
 import org.jitsi.utils.logging.TimeSeriesLogger
 import org.jitsi.utils.logging2.Logger
+import org.jitsi.utils.logging2.createChildLogger
 import org.jitsi.utils.secs
 import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config
 import org.jitsi.videobridge.message.ReceiverVideoConstraintsMessage
@@ -53,6 +54,8 @@ class BitrateController<T : MediaSourceContainer> @JvmOverloads constructor(
 ) {
     val eventEmitter = SyncEventEmitter<EventHandler>()
 
+    private val logger = createChildLogger(parentLogger)
+
     private val bitrateAllocatorEventHandler = BitrateAllocatorEventHandler()
 
     /**
@@ -133,6 +136,9 @@ class BitrateController<T : MediaSourceContainer> @JvmOverloads constructor(
     fun accept(packetInfo: PacketInfo): Boolean {
         if (packetInfo.layeringChanged) {
             // This needs to be done synchronously, so it's complete before the accept, below.
+            logger.debug {
+                "Layering information changed for packet from ${packetInfo.endpointId}, updating bandwidth allocation"
+            }
             bandwidthAllocator.update()
         }
         return packetHandler.accept(packetInfo)

jvb/src/main/kotlin/org/jitsi/videobridge/cc/allocation/SingleSourceAllocation.kt

@@ -26,6 +26,7 @@ import org.jitsi.utils.logging2.LoggerImpl
 import org.jitsi.videobridge.cc.config.BitrateControllerConfig.Companion.config
 import java.lang.Integer.max
 import java.time.Clock
+import kotlin.math.min
 
 /**
  * A bitrate allocation that pertains to a specific source. This is the internal representation used in the allocation
@@ -68,6 +69,7 @@ internal class SingleSourceAllocation(
             }
             timeSeriesLogger.trace(ratesTimeSeriesPoint)
         }
+        logger.addContext(mapOf("remote_endpoint_id" to endpointId))
     }
 
     fun isOnStage() = onStage
@@ -294,9 +296,15 @@ internal class SingleSourceAllocation(
      * oversending.
      */
     private fun selectLayersForCamera(layers: List<LayerSnapshot>, constraints: VideoConstraints): Layers {
-        val minHeight = layers.map { it.layer.height }.minOrNull() ?: return Layers.noLayers
+        val minHeight = layers.minOfOrNull { it.layer.height } ?: return Layers.noLayers
+        val maxFps = layers.maxOfOrNull { it.layer.frameRate } ?: return Layers.noLayers
         val noActiveLayers = layers.none { (_, bitrate) -> bitrate > 0 }
         val (preferredHeight, preferredFps) = getPreferred(constraints)
+        val effectivePreferredFps = if (maxFps > 0) {
+            min(maxFps, preferredFps)
+        } else {
+            preferredFps
+        }
 
         val ratesList: MutableList<LayerSnapshot> = ArrayList()
         // Initialize the list of layers to be considered. These are the layers that satisfy the constraints, with
@@ -306,7 +314,7 @@ internal class SingleSourceAllocation(
             val lessThanPreferredHeight = layer.height < preferredHeight
             val lessThanOrEqualMaxHeight = layer.height <= constraints.maxHeight || !constraints.heightIsLimited()
             // If frame rate is unknown, consider it to be sufficient.
-            val atLeastPreferredFps = layer.frameRate < 0 || layer.frameRate >= preferredFps
+            val atLeastPreferredFps = layer.frameRate < 0 || layer.frameRate >= effectivePreferredFps
             if (lessThanPreferredHeight ||
                 (lessThanOrEqualMaxHeight && atLeastPreferredFps) ||
                 layer.height == minHeight
@@ -321,6 +329,11 @@ internal class SingleSourceAllocation(
 
         val effectivePreferredHeight = max(preferredHeight, minHeight)
         val preferredIndex = ratesList.lastIndexWhich { it.layer.height <= effectivePreferredHeight }
+        logger.trace {
+            "Selected rates list $ratesList, preferred index $preferredIndex " +
+                "from layers $layers with constraints $constraints"
+        }
+
         return Layers(ratesList, preferredIndex, -1)
     }