From 0892536b2bade80e950d6229f31b280cf2c1df27 Mon Sep 17 00:00:00 2001 From: burak-58 Date: Tue, 11 Apr 2023 20:30:57 +0300 Subject: [PATCH] don't wait video frame for amcu --- .../io/antmedia/filter/FilterAdaptor.java | 69 ++++++----- .../antmedia/test/FilterAdaptorUnitTest.java | 109 +++++++++++------- 2 files changed, 111 insertions(+), 67 deletions(-) diff --git a/FilterPlugin/src/main/java/io/antmedia/filter/FilterAdaptor.java b/FilterPlugin/src/main/java/io/antmedia/filter/FilterAdaptor.java index 423ee401..6679e797 100644 --- a/FilterPlugin/src/main/java/io/antmedia/filter/FilterAdaptor.java +++ b/FilterPlugin/src/main/java/io/antmedia/filter/FilterAdaptor.java @@ -333,18 +333,7 @@ public Result update() * Set the listener of video filter graph. FilterGrapah calls the listener for the filtered output frame */ videoFilterGraph.setListener((streamId, frame)->{ - if(frame != null && currentOutStreams.containsKey(streamId)) { - IFrameListener frameListener = currentOutStreams.get(streamId); - if(frameListener != null) { - if(!firstVideoReceived) { - firstVideoReceived = true; - } - //rescale the pts if the filter timebase is different - frame.pts(av_rescale_q_rnd(frame.pts(), videoSinkFiltersMap.get(streamId).getFilterContext().inputs(0).time_base(), Utils.TIME_BASE_FOR_MS, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX)); - //framelistener is a custombroadcast - frameListener.onVideoFrame(streamId, frame); - } - } + onFilteredVideoFrame(videoSinkFiltersMap, streamId, frame); }); if(prevVideoFilterGraph != null) { @@ -368,22 +357,7 @@ public Result update() return result; } audioFilterGraph.setCurrentPts(currentAudioPts); - audioFilterGraph.setListener((streamId, frame)->{ - if(frame != null && currentOutStreams.containsKey(streamId)) { - IFrameListener frameListener = currentOutStreams.get(streamId); - if(frameListener != null) { - if(!firstVideoReceived) { - audioVideoOffset = frame.pts(); - } - else { - frame.pts(frame.pts()-audioVideoOffset); - - //framelistener is a custombroadcast - frameListener.onAudioFrame(streamId, frame); - } - } - } - }); + audioFilterGraph.setListener(this::onFilteredAudioFrame); if(prevAudioFilterGraph != null) { prevAudioFilterGraph.close(); @@ -394,6 +368,42 @@ public Result update() return result; } + public void onFilteredVideoFrame(Map videoSinkFiltersMap, String streamId, AVFrame frame) { + if(frame != null && currentOutStreams.containsKey(streamId)) { + IFrameListener frameListener = currentOutStreams.get(streamId); + if(frameListener != null) { + if(!firstVideoReceived) { + firstVideoReceived = true; + } + //rescale the pts if the filter timebase is different + frame.pts(av_rescale_q_rnd(frame.pts(), getFilterTimebase(videoSinkFiltersMap, streamId), Utils.TIME_BASE_FOR_MS, AV_ROUND_NEAR_INF|AV_ROUND_PASS_MINMAX)); + //framelistener is a custombroadcast + frameListener.onVideoFrame(streamId, frame); + } + } + } + + public AVRational getFilterTimebase(Map videoSinkFiltersMap, String streamId) { + return videoSinkFiltersMap.get(streamId).getFilterContext().inputs(0).time_base(); + } + + public void onFilteredAudioFrame(String streamId, AVFrame frame) { + if(frame != null && currentOutStreams.containsKey(streamId)) { + IFrameListener frameListener = currentOutStreams.get(streamId); + if(frameListener != null) { + if(filterConfiguration.isVideoEnabled() && !firstVideoReceived) { + audioVideoOffset = frame.pts(); + } + else { + frame.pts(frame.pts()-audioVideoOffset); + + //framelistener is a custombroadcast + frameListener.onAudioFrame(streamId, frame); + } + } + } + } + /* * This method is used for the creation and also for the update of the filter * For example new inputs mat be added as an update @@ -604,4 +614,7 @@ public void setFilterConfiguration(FilterConfiguration filterConfiguration) { this.filterConfiguration = filterConfiguration; } + public Map getCurrentOutStreams() { + return currentOutStreams; + } } diff --git a/FilterPlugin/src/test/java/io/antmedia/test/FilterAdaptorUnitTest.java b/FilterPlugin/src/test/java/io/antmedia/test/FilterAdaptorUnitTest.java index 4e64fa40..26633596 100644 --- a/FilterPlugin/src/test/java/io/antmedia/test/FilterAdaptorUnitTest.java +++ b/FilterPlugin/src/test/java/io/antmedia/test/FilterAdaptorUnitTest.java @@ -6,26 +6,24 @@ import static org.mockito.ArgumentMatchers.anyBoolean; import static org.mockito.ArgumentMatchers.anyString; import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.Mockito.doNothing; -import static org.mockito.Mockito.doReturn; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.never; -import static org.mockito.Mockito.spy; -import static org.mockito.Mockito.timeout; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import static org.mockito.Mockito.*; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import io.antmedia.filter.utils.Filter; import org.apache.commons.lang3.RandomStringUtils; import org.apache.commons.lang3.RandomUtils; import org.apache.commons.lang3.exception.ExceptionUtils; import org.bytedeco.ffmpeg.avcodec.AVCodecParameters; +import org.bytedeco.ffmpeg.avfilter.AVFilterContext; import org.bytedeco.ffmpeg.avutil.AVChannelLayout; import org.bytedeco.ffmpeg.avutil.AVFrame; import org.bytedeco.ffmpeg.avutil.AVRational; import org.bytedeco.ffmpeg.global.avutil; +import org.bytedeco.javacpp.PointerPointer; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Rule; @@ -47,7 +45,7 @@ import io.vertx.core.Vertx; public class FilterAdaptorUnitTest { - + @Rule public TestRule watcher = new TestWatcher() { protected void starting(Description description) { @@ -62,7 +60,7 @@ protected void finished(Description description) { System.out.println("Finishing test: " + description.getMethodName()); }; }; - + private static Vertx vertx; @AfterClass @@ -75,7 +73,7 @@ public static void beforeClass() { vertx = Vertx.vertx(); } - + @Test public void testFilterGraphVideoFeed() { FilterAdaptor filterAdaptor = spy(new FilterAdaptor(RandomStringUtils.randomAlphanumeric(12), false)); @@ -87,10 +85,10 @@ public void testFilterGraphVideoFeed() { AntMediaApplicationAdapter app = mock(AntMediaApplicationAdapter.class); when(app.getVertx()).thenReturn(vertx); - + filterAdaptor.createOrUpdateFilter(filterConf, app); - + String streamId = "stream"+RandomUtils.nextInt(0, 10000); AVFrame frame = new AVFrame(); FilterGraph filterGraph = mock(FilterGraph.class); @@ -114,12 +112,12 @@ public void testFilterGraphVideoFeed() { vsi.setCodecParameters(mock(AVCodecParameters.class)); vsi.setTimeBase(Utils.TIME_BASE_FOR_MS); filterAdaptor.setVideoStreamInfo(streamId, vsi); - + filterAdaptor.onVideoFrame(streamId, frame); verify(filterGraph, timeout(3000)).doFilter(eq(streamId), any(), Mockito.anyBoolean()); } - - + + @Test public void testFilterGraphAudioFeed() { FilterAdaptor filterAdaptor = spy(new FilterAdaptor(RandomStringUtils.randomAlphanumeric(12), false)); @@ -130,10 +128,10 @@ public void testFilterGraphAudioFeed() { AntMediaApplicationAdapter app = mock(AntMediaApplicationAdapter.class); when(app.getVertx()).thenReturn(vertx); - + filterAdaptor.createOrUpdateFilter(filterConf, app); - + String streamId = "stream"+RandomUtils.nextInt(0, 10000); AVFrame frame = new AVFrame(); FilterGraph filterGraph = mock(FilterGraph.class); @@ -156,29 +154,29 @@ public void testFilterGraphAudioFeed() { filterAdaptor.onAudioFrame(streamId, frame); verify(filterGraph, timeout(3000)).doFilter(eq(streamId), any(), anyBoolean()); } - + @Test public void testVideoAudioFiltering() { testFiltering(true, "[in0][in1][in2]vstack=inputs=3[out0]", true, "[in0][in1][in2]amix=inputs=3[out0]"); } - + //use some of the inputs in the filter - @Test + @Test public void testPartialVideoAudioFiltering() { testFiltering(true, "[in0][in2]vstack=inputs=2[out0]", true, "[in0][in2]amix=inputs=2[out0]"); } - + @Test public void testVideoOnlyFiltering() { testFiltering(true, "[in0][in1][in2]vstack=inputs=3[out0]", false, "dummy"); } - + @Test public void testAudioOnlyFiltering() { testFiltering(false, "dummy", true, "[in0][in1][in2]amix=inputs=3[out0]"); } - - + + public void testFiltering(boolean videoEnabled, String videoFilter, boolean audioEnabled, String audioFilter) { FilterAdaptor filterAdaptor = spy(new FilterAdaptor(RandomStringUtils.randomAlphanumeric(12), false)); AntMediaApplicationAdapter app = mock(AntMediaApplicationAdapter.class); @@ -187,7 +185,7 @@ public void testFiltering(boolean videoEnabled, String videoFilter, boolean audi String stream1 = "inStream1"; String stream2 = "inStream2"; String stream3 = "inStream3"; - + String output1 = "outStream1"; @@ -199,16 +197,16 @@ public void testFiltering(boolean videoEnabled, String videoFilter, boolean audi StreamParametersInfo asi2 = getStreamInfo(); StreamParametersInfo asi3 = getStreamInfo(); - + filterAdaptor.setVideoStreamInfo(stream1, vsi1); filterAdaptor.setAudioStreamInfo(stream1, asi1); - + filterAdaptor.setVideoStreamInfo(stream2, vsi2); filterAdaptor.setAudioStreamInfo(stream2, asi2); - + filterAdaptor.setVideoStreamInfo(stream3, vsi3); filterAdaptor.setAudioStreamInfo(stream3, asi3); - + FilterConfiguration conf = new FilterConfiguration(); conf.setVideoEnabled(videoEnabled); conf.setAudioEnabled(audioEnabled); @@ -216,14 +214,14 @@ public void testFiltering(boolean videoEnabled, String videoFilter, boolean audi conf.setAudioFilter(audioFilter); conf.setInputStreams(Arrays.asList(stream1, stream2, stream3)); conf.setOutputStreams(Arrays.asList(output1)); - + assertTrue(filterAdaptor.createOrUpdateFilter(conf, app).isSuccess()); - + filterAdaptor.close(app); - + //increase coverage and checking not throwing exception filterAdaptor.close(app); - + } public StreamParametersInfo getStreamInfo() { @@ -231,7 +229,7 @@ public StreamParametersInfo getStreamInfo() { AVCodecParameters codecParams = mock(AVCodecParameters.class); when(codecParams.height()).thenReturn(360); when(codecParams.width()).thenReturn(640); - + AVChannelLayout channelLayout = new AVChannelLayout(); avutil.av_channel_layout_default(channelLayout, 2); when(codecParams.ch_layout()).thenReturn(channelLayout); @@ -242,16 +240,16 @@ public StreamParametersInfo getStreamInfo() { when(tb.num()).thenReturn(1); when(tb.den()).thenReturn(1000); - + si.setEnabled(true); si.setCodecParameters(codecParams); - - + + si.setTimeBase(tb); return si; } - + /* * In synchronous mode output frame pts should be the same with input, * because we apply filter on going original stream without creating a new stream. @@ -311,4 +309,37 @@ public void testTimeBaseInSyncMode() { assertEquals(filteredFrame.pts(), frame.pts()); } } + + @Test + public void testOnFilteredAudioFrame() { + onFilteredAudioFrameTest(true); + onFilteredAudioFrameTest(false); + } + + public void onFilteredAudioFrameTest(boolean videoEnabled) { + + FilterAdaptor filterAdaptor = spy(new FilterAdaptor(RandomStringUtils.randomAlphanumeric(12), false)); + String streamId = "stream"+RandomUtils.nextInt(0, 10000); + IFrameListener frameListener = mock(IFrameListener.class); + filterAdaptor.getCurrentOutStreams().put(streamId, frameListener); + AVFrame frame = mock(AVFrame.class); + when(frame.pts(anyLong())).thenReturn(frame); + + FilterConfiguration filterConfiguration = new FilterConfiguration(); + filterConfiguration.setAudioEnabled(true); + filterConfiguration.setVideoEnabled(videoEnabled); + filterAdaptor.setFilterConfiguration(filterConfiguration); + + Map videoSinkFiltersMap = new HashMap<>(); + Filter filter = mock(Filter.class); + doReturn(Utils.TIME_BASE_FOR_MS).when(filterAdaptor).getFilterTimebase(videoSinkFiltersMap, streamId); + videoSinkFiltersMap.put(streamId, filter); + for (int i = 0; i < 20; i++) { + if(i==10) { + filterAdaptor.onFilteredVideoFrame(videoSinkFiltersMap, streamId, frame); + } + filterAdaptor.onFilteredAudioFrame(streamId, frame); + } + verify(frameListener, times(videoEnabled?10:20)).onAudioFrame(streamId, frame); + } }