Fix enhanced RTMP video codec detection

Author: mondain

common/src/main/java/org/red5/server/stream/VideoCodecFactory.java

@@ -8,9 +8,10 @@
 package org.red5.server.stream;
 
 import org.apache.mina.core.buffer.IoBuffer;
-import org.red5.codec.AbstractVideo;
 import org.red5.codec.IVideoStreamCodec;
 import org.red5.codec.VideoCodec;
+import org.red5.codec.VideoFrameType;
+import org.red5.codec.VideoPacketType;
 import org.red5.io.IoConstants;
 import org.red5.util.ByteNibbler;
 import org.slf4j.Logger;
@@ -53,9 +54,7 @@ public static IVideoStreamCodec getVideoCodec(IoBuffer data) {
             boolean enhanced = ByteNibbler.isBitSet(c, 7);
             if (enhanced) {
                 log.debug("Enhanced codec handling; pos: {}", data.position());
-                AbstractVideo absv = new AbstractVideo();
-                absv.addData(data, 0);
-                codec = absv.getTrackCodec(0);
+                codec = getEnhancedVideoCodec(data);
             } else {
                 int codecId = (c & IoConstants.MASK_VIDEO_CODEC);
                 codec = VideoCodec.valueOfById(codecId);
@@ -84,4 +83,29 @@ public static IVideoStreamCodec getVideoCodec(IoBuffer data) {
         return result;
     }
 
+    private static VideoCodec getEnhancedVideoCodec(IoBuffer data) {
+        VideoCodec codec = null;
+        data.mark();
+        try {
+            byte flags = data.get();
+            VideoFrameType frameType = VideoFrameType.valueOf((flags & IoConstants.MASK_VIDEO_FRAMETYPE) >> 4);
+            VideoPacketType packetType = VideoPacketType.valueOf(flags & IoConstants.MASK_VIDEO_CODEC);
+            if (frameType == VideoFrameType.COMMAND_FRAME && packetType != VideoPacketType.Metadata) {
+                return null;
+            }
+            if (packetType == VideoPacketType.Multitrack) {
+                if (!data.hasRemaining()) {
+                    return null;
+                }
+                data.skip(1);
+            }
+            if (data.remaining() >= Integer.BYTES) {
+                codec = VideoCodec.valueOfByFourCc(data.getInt());
+            }
+        } finally {
+            data.reset();
+        }
+        return codec;
+    }
+
 }

common/src/test/java/org/red5/server/stream/VideoCodecFactoryTest.java

@@ -0,0 +1,56 @@
+package org.red5.server.stream;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.apache.mina.core.buffer.IoBuffer;
+import org.junit.Test;
+import org.red5.codec.AbstractVideo;
+import org.red5.codec.IVideoStreamCodec;
+import org.red5.codec.VideoCodec;
+import org.red5.io.utils.IOUtils;
+
+import ch.qos.logback.classic.Level;
+import ch.qos.logback.classic.Logger;
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.read.ListAppender;
+
+public class VideoCodecFactoryTest {
+
+    @Test
+    public void testEnhancedAv1SequenceStartDoesNotUseRejectingProbeCodec() {
+        assertEnhancedSequenceStartDoesNotUseRejectingProbeCodec("av01", VideoCodec.AV1, new byte[] { (byte) 0x81, 0x08, 0x0d });
+    }
+
+    @Test
+    public void testEnhancedHevcSequenceStartDoesNotUseRejectingProbeCodec() {
+        assertEnhancedSequenceStartDoesNotUseRejectingProbeCodec("hvc1", VideoCodec.HEVC, new byte[] { 0x01, 0x01, 0x60, 0x00 });
+    }
+
+    private static void assertEnhancedSequenceStartDoesNotUseRejectingProbeCodec(String fourcc, VideoCodec expectedCodec, byte[] configPrefix) {
+        Logger logger = (Logger) org.slf4j.LoggerFactory.getLogger(AbstractVideo.class);
+        ListAppender<ILoggingEvent> appender = new ListAppender<>();
+        appender.start();
+        logger.addAppender(appender);
+        try {
+            IoBuffer data = IoBuffer.allocate(16);
+            data.put((byte) 0x90);
+            data.putInt(IOUtils.makeFourcc(fourcc));
+            data.put(configPrefix);
+            data.flip();
+
+            IVideoStreamCodec codec = VideoCodecFactory.getVideoCodec(data);
+
+            assertEquals(expectedCodec, codec.getCodec());
+            assertTrue("Codec factory must not emit AbstractVideo rejection while probing enhanced FourCC", rejectLogs(appender.list).isEmpty());
+        } finally {
+            logger.detachAppender(appender);
+        }
+    }
+
+    private static List<ILoggingEvent> rejectLogs(List<ILoggingEvent> events) {
+        return events.stream().filter(event -> event.getLevel().isGreaterOrEqual(Level.WARN)).filter(event -> event.getFormattedMessage().contains("AbstractVideo rejected")).toList();
+    }
+}