diff --git a/core/src/display_object/video.rs b/core/src/display_object/video.rs index 4dd99997b7b7..047de01ca473 100644 --- a/core/src/display_object/video.rs +++ b/core/src/display_object/video.rs @@ -312,6 +312,7 @@ impl<'gc> Video<'gc> { codec: streamdef.codec, data: &read.movie.data()[*slice_start..*slice_end], frame_id, + composition_time: None, }; context .video @@ -409,6 +410,7 @@ impl<'gc> TDisplayObject<'gc> for Video<'gc> { codec: streamdef.codec, data: &movie.data()[*frame_start..*frame_end], frame_id: *frame_id, + composition_time: None, }, ); diff --git a/core/src/streams.rs b/core/src/streams.rs index 831d990c3a86..d2187100b5ee 100644 --- a/core/src/streams.rs +++ b/core/src/streams.rs @@ -947,6 +947,7 @@ impl<'gc> NetStream<'gc> { codec, data, //TODO: ScreenVideo's decoder wants the FLV header bytes frame_id, + composition_time: Some(write.stream_time as i32), }; if let Err(e) = context @@ -961,6 +962,7 @@ impl<'gc> NetStream<'gc> { codec, data, //TODO: ScreenVideo's decoder wants the FLV header bytes frame_id, + composition_time: Some(write.stream_time as i32), }; match context.video.decode_video_stream_frame( @@ -998,7 +1000,7 @@ impl<'gc> NetStream<'gc> { Some(video_handle), Some(codec), FlvVideoPacket::AvcNalu { - composition_time_offset: _, + composition_time_offset, data, }, ) => { @@ -1006,6 +1008,7 @@ impl<'gc> NetStream<'gc> { codec, data, frame_id, + composition_time: Some(write.stream_time as i32 + composition_time_offset), }; match context.video.decode_video_stream_frame( diff --git a/video/external/src/decoder/openh264.rs b/video/external/src/decoder/openh264.rs index 5091c6262853..d8e682ac5400 100644 --- a/video/external/src/decoder/openh264.rs +++ b/video/external/src/decoder/openh264.rs @@ -1,4 +1,5 @@ use core::slice; +use std::collections::BinaryHeap; use std::ffi::{c_int, c_uchar}; use std::fmt::Display; use std::fs::File; @@ -192,6 +193,33 @@ impl OpenH264Codec { } } +struct EnqueuedFrame { + composition_time: i32, + frame: DecodedFrame, +} + +impl PartialEq for EnqueuedFrame { + fn eq(&self, other: &Self) -> bool { + self.composition_time == other.composition_time + } +} + +impl Eq for EnqueuedFrame {} + +impl Ord for EnqueuedFrame { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // Note the reversal: BinaryHeap is a max-heap, but we always + // want to pop the frame with the lowest timestamp. + self.composition_time.cmp(&other.composition_time).reverse() + } +} + +impl PartialOrd for EnqueuedFrame { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + /// H264 video decoder. pub struct H264Decoder { /// How many bytes are used to store the length of the NALU (1, 2, 3, or 4). @@ -199,6 +227,8 @@ pub struct H264Decoder { openh264: Arc, decoder: *mut ISVCDecoder, + + frame_reorder_queue: BinaryHeap, } struct OpenH264Data { @@ -227,6 +257,7 @@ impl H264Decoder { length_size: 0, openh264, decoder, + frame_reorder_queue: BinaryHeap::new(), } } } @@ -293,15 +324,22 @@ impl VideoDecoder for H264Decoder { //in-out: for Decoding only: declare and initialize the output buffer info let mut dest_buf_info: openh264_sys::SBufferInfo = std::mem::zeroed(); - let _ret = decoder_vtbl.DecodeFrameNoDelay.unwrap()( + let ret = decoder_vtbl.DecodeFrameNoDelay.unwrap()( self.decoder, buffer.as_mut_ptr(), buffer.len() as c_int, output.as_mut_ptr(), &mut dest_buf_info as *mut openh264_sys::SBufferInfo, ); + + if ret == 0 { + Ok(()) + } else { + Err(Error::DecoderError( + format!("Configuration failed with status code: {}", ret).into(), + )) + } } - Ok(()) } fn preload_frame(&mut self, encoded_frame: EncodedFrame<'_>) -> Result { @@ -359,55 +397,77 @@ impl VideoDecoder for H264Decoder { )); } if dest_buf_info.iBufferStatus != 1 { - return Err(Error::DecoderError( - "No output frame produced by the decoder".into(), - )); - } - let buffer_info = dest_buf_info.UsrData.sSystemBuffer; - if buffer_info.iFormat != videoFormatI420 as c_int { - return Err(Error::DecoderError( - format!("Unexpected output format: {}", buffer_info.iFormat).into(), - )); + let ret = decoder_vtbl.FlushFrame.unwrap()( + self.decoder, + output.as_mut_ptr(), + &mut dest_buf_info as *mut openh264_sys::SBufferInfo, + ); + + if ret != 0 { + return Err(Error::DecoderError( + format!("Flushing failed with status code: {}", ret).into(), + )); + } } - let mut yuv: Vec = Vec::with_capacity( - buffer_info.iWidth as usize * buffer_info.iHeight as usize * 3 / 2, - ); + if dest_buf_info.iBufferStatus == 1 { + let buffer_info = dest_buf_info.UsrData.sSystemBuffer; + if buffer_info.iFormat != videoFormatI420 as c_int { + return Err(Error::DecoderError( + format!("Unexpected output format: {}", buffer_info.iFormat).into(), + )); + } - // Copying Y - for i in 0..buffer_info.iHeight { - yuv.extend_from_slice(slice::from_raw_parts( - output[0].offset((i * buffer_info.iStride[0]) as isize), - buffer_info.iWidth as usize, - )); - } + let mut yuv: Vec = Vec::with_capacity( + buffer_info.iWidth as usize * buffer_info.iHeight as usize * 3 / 2, + ); - // Copying U - for i in 0..buffer_info.iHeight / 2 { - yuv.extend_from_slice(slice::from_raw_parts( - output[1].offset((i * buffer_info.iStride[1]) as isize), - buffer_info.iWidth as usize / 2, - )); - } + // Copying Y + for i in 0..buffer_info.iHeight { + yuv.extend_from_slice(slice::from_raw_parts( + output[0].offset((i * buffer_info.iStride[0]) as isize), + buffer_info.iWidth as usize, + )); + } - // Copying V - for i in 0..buffer_info.iHeight / 2 { - yuv.extend_from_slice(slice::from_raw_parts( - output[2].offset((i * buffer_info.iStride[1]) as isize), - buffer_info.iWidth as usize / 2, - )); + // Copying U + for i in 0..buffer_info.iHeight / 2 { + yuv.extend_from_slice(slice::from_raw_parts( + output[1].offset((i * buffer_info.iStride[1]) as isize), + buffer_info.iWidth as usize / 2, + )); + } + + // Copying V + for i in 0..buffer_info.iHeight / 2 { + yuv.extend_from_slice(slice::from_raw_parts( + output[2].offset((i * buffer_info.iStride[1]) as isize), + buffer_info.iWidth as usize / 2, + )); + } + + // TODO: Check whether frames are being squished/stretched, or cropped, + // when encoded image size doesn't match declared video tag size. + // NOTE: This will always use the BT.601 coefficients, which may or may + // not be correct. So far I haven't seen anything to the contrary in FP. + self.frame_reorder_queue.push(EnqueuedFrame { + composition_time: encoded_frame + .composition_time + .ok_or(Error::DecoderError("No composition time provided".into()))?, + frame: DecodedFrame::new( + buffer_info.iWidth as u32, + buffer_info.iHeight as u32, + BitmapFormat::Yuv420p, + yuv, + ), + }); } - // TODO: Check whether frames are being squished/stretched, or cropped, - // when encoded image size doesn't match declared video tag size. - // NOTE: This will always use the BT.601 coefficients, which may or may - // not be correct. So far I haven't seen anything to the contrary in FP. - Ok(DecodedFrame::new( - buffer_info.iWidth as u32, - buffer_info.iHeight as u32, - BitmapFormat::Yuv420p, - yuv, - )) + if self.frame_reorder_queue.len() >= 3 { + Ok(self.frame_reorder_queue.pop().unwrap().frame) + } else { + Err(Error::DecoderError("Not enough frames decoded yet".into())) + } } } } diff --git a/video/src/frame.rs b/video/src/frame.rs index 00dd0f1efa52..19b71b1d82a3 100644 --- a/video/src/frame.rs +++ b/video/src/frame.rs @@ -12,6 +12,8 @@ pub struct EncodedFrame<'a> { /// A caller-specified frame ID. Frame IDs must be consistent between /// subsequent uses of the same data stream. pub frame_id: u32, + + pub composition_time: Option, } impl<'a> EncodedFrame<'a> {