Skip to content

Commit 31d87b9

Browse files
torokati44torokati44
committed
video/external: Reorder frames decoded by OpenH264 manually
1 parent 41e4c65 commit 31d87b9

File tree

4 files changed

+102
-43
lines changed

4 files changed

+102
-43
lines changed

‎core/src/display_object/video.rs

+2
Original file line numberDiff line numberDiff line change
@@ -312,6 +312,7 @@ impl<'gc> Video<'gc> {
312312
codec: streamdef.codec,
313313
data: &read.movie.data()[*slice_start..*slice_end],
314314
frame_id,
315+
composition_time: None,
315316
};
316317
context
317318
.video
@@ -409,6 +410,7 @@ impl<'gc> TDisplayObject<'gc> for Video<'gc> {
409410
codec: streamdef.codec,
410411
data: &movie.data()[*frame_start..*frame_end],
411412
frame_id: *frame_id,
413+
composition_time: None,
412414
},
413415
);
414416

‎core/src/streams.rs

+4-1
Original file line numberDiff line numberDiff line change
@@ -947,6 +947,7 @@ impl<'gc> NetStream<'gc> {
947947
codec,
948948
data, //TODO: ScreenVideo's decoder wants the FLV header bytes
949949
frame_id,
950+
composition_time: Some(write.stream_time as i32),
950951
};
951952

952953
if let Err(e) = context
@@ -961,6 +962,7 @@ impl<'gc> NetStream<'gc> {
961962
codec,
962963
data, //TODO: ScreenVideo's decoder wants the FLV header bytes
963964
frame_id,
965+
composition_time: Some(write.stream_time as i32),
964966
};
965967

966968
match context.video.decode_video_stream_frame(
@@ -998,14 +1000,15 @@ impl<'gc> NetStream<'gc> {
9981000
Some(video_handle),
9991001
Some(codec),
10001002
FlvVideoPacket::AvcNalu {
1001-
composition_time_offset: _,
1003+
composition_time_offset,
10021004
data,
10031005
},
10041006
) => {
10051007
let encoded_frame = EncodedFrame {
10061008
codec,
10071009
data,
10081010
frame_id,
1011+
composition_time: Some(write.stream_time as i32 + composition_time_offset),
10091012
};
10101013

10111014
match context.video.decode_video_stream_frame(

‎video/external/src/decoder/openh264.rs

+94-42
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
use core::slice;
2+
use std::cmp::Ordering;
3+
use std::collections::BinaryHeap;
24
use std::ffi::{c_int, c_uchar};
35
use std::fmt::Display;
46
use std::fs::File;
@@ -192,13 +194,40 @@ impl OpenH264Codec {
192194
}
193195
}
194196

197+
struct EnqueuedFrame {
198+
composition_time: i32,
199+
frame: DecodedFrame,
200+
}
201+
202+
impl PartialEq for EnqueuedFrame {
203+
fn eq(&self, other: &Self) -> bool {
204+
self.composition_time == other.composition_time
205+
}
206+
}
207+
208+
impl Eq for EnqueuedFrame {}
209+
210+
impl Ord for EnqueuedFrame {
211+
fn cmp(&self, other: &Self) -> std::cmp::Ordering {
212+
self.composition_time.cmp(&other.composition_time).reverse()
213+
}
214+
}
215+
216+
impl PartialOrd for EnqueuedFrame {
217+
fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
218+
Some(self.cmp(other)) // Directly use `cmp`
219+
}
220+
}
221+
195222
/// H264 video decoder.
196223
pub struct H264Decoder {
197224
/// How many bytes are used to store the length of the NALU (1, 2, 3, or 4).
198225
length_size: u8,
199226

200227
openh264: Arc<OpenH264>,
201228
decoder: *mut ISVCDecoder,
229+
230+
frame_reorder_queue: BinaryHeap<EnqueuedFrame>,
202231
}
203232

204233
struct OpenH264Data {
@@ -227,6 +256,7 @@ impl H264Decoder {
227256
length_size: 0,
228257
openh264,
229258
decoder,
259+
frame_reorder_queue: BinaryHeap::new(),
230260
}
231261
}
232262
}
@@ -366,55 +396,77 @@ impl VideoDecoder for H264Decoder {
366396
));
367397
}
368398
if dest_buf_info.iBufferStatus != 1 {
369-
return Err(Error::DecoderError(
370-
"No output frame produced by the decoder".into(),
371-
));
372-
}
373-
let buffer_info = dest_buf_info.UsrData.sSystemBuffer;
374-
if buffer_info.iFormat != videoFormatI420 as c_int {
375-
return Err(Error::DecoderError(
376-
format!("Unexpected output format: {}", buffer_info.iFormat).into(),
377-
));
399+
let ret = decoder_vtbl.FlushFrame.unwrap()(
400+
self.decoder,
401+
output.as_mut_ptr(),
402+
&mut dest_buf_info as *mut openh264_sys::SBufferInfo,
403+
);
404+
405+
if ret != 0 {
406+
return Err(Error::DecoderError(
407+
format!("Flushing failed with status code: {}", ret).into(),
408+
));
409+
}
378410
}
379411

380-
let mut yuv: Vec<u8> = Vec::with_capacity(
381-
buffer_info.iWidth as usize * buffer_info.iHeight as usize * 3 / 2,
382-
);
412+
if dest_buf_info.iBufferStatus == 1 {
413+
let buffer_info = dest_buf_info.UsrData.sSystemBuffer;
414+
if buffer_info.iFormat != videoFormatI420 as c_int {
415+
return Err(Error::DecoderError(
416+
format!("Unexpected output format: {}", buffer_info.iFormat).into(),
417+
));
418+
}
383419

384-
// Copying Y
385-
for i in 0..buffer_info.iHeight {
386-
yuv.extend_from_slice(slice::from_raw_parts(
387-
output[0].offset((i * buffer_info.iStride[0]) as isize),
388-
buffer_info.iWidth as usize,
389-
));
390-
}
420+
let mut yuv: Vec<u8> = Vec::with_capacity(
421+
buffer_info.iWidth as usize * buffer_info.iHeight as usize * 3 / 2,
422+
);
391423

392-
// Copying U
393-
for i in 0..buffer_info.iHeight / 2 {
394-
yuv.extend_from_slice(slice::from_raw_parts(
395-
output[1].offset((i * buffer_info.iStride[1]) as isize),
396-
buffer_info.iWidth as usize / 2,
397-
));
398-
}
424+
// Copying Y
425+
for i in 0..buffer_info.iHeight {
426+
yuv.extend_from_slice(slice::from_raw_parts(
427+
output[0].offset((i * buffer_info.iStride[0]) as isize),
428+
buffer_info.iWidth as usize,
429+
));
430+
}
399431

400-
// Copying V
401-
for i in 0..buffer_info.iHeight / 2 {
402-
yuv.extend_from_slice(slice::from_raw_parts(
403-
output[2].offset((i * buffer_info.iStride[1]) as isize),
404-
buffer_info.iWidth as usize / 2,
405-
));
432+
// Copying U
433+
for i in 0..buffer_info.iHeight / 2 {
434+
yuv.extend_from_slice(slice::from_raw_parts(
435+
output[1].offset((i * buffer_info.iStride[1]) as isize),
436+
buffer_info.iWidth as usize / 2,
437+
));
438+
}
439+
440+
// Copying V
441+
for i in 0..buffer_info.iHeight / 2 {
442+
yuv.extend_from_slice(slice::from_raw_parts(
443+
output[2].offset((i * buffer_info.iStride[1]) as isize),
444+
buffer_info.iWidth as usize / 2,
445+
));
446+
}
447+
448+
// TODO: Check whether frames are being squished/stretched, or cropped,
449+
// when encoded image size doesn't match declared video tag size.
450+
// NOTE: This will always use the BT.601 coefficients, which may or may
451+
// not be correct. So far I haven't seen anything to the contrary in FP.
452+
self.frame_reorder_queue.push(EnqueuedFrame {
453+
composition_time: encoded_frame
454+
.composition_time
455+
.ok_or(Error::DecoderError("No composition time provided".into()))?,
456+
frame: DecodedFrame::new(
457+
buffer_info.iWidth as u32,
458+
buffer_info.iHeight as u32,
459+
BitmapFormat::Yuv420p,
460+
yuv,
461+
),
462+
});
406463
}
407464

408-
// TODO: Check whether frames are being squished/stretched, or cropped,
409-
// when encoded image size doesn't match declared video tag size.
410-
// NOTE: This will always use the BT.601 coefficients, which may or may
411-
// not be correct. So far I haven't seen anything to the contrary in FP.
412-
Ok(DecodedFrame::new(
413-
buffer_info.iWidth as u32,
414-
buffer_info.iHeight as u32,
415-
BitmapFormat::Yuv420p,
416-
yuv,
417-
))
465+
if self.frame_reorder_queue.len() >= 3 {
466+
Ok(self.frame_reorder_queue.pop().unwrap().frame)
467+
} else {
468+
Err(Error::DecoderError("Not enough frames decoded yet".into()))
469+
}
418470
}
419471
}
420472
}

‎video/src/frame.rs

+2
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ pub struct EncodedFrame<'a> {
1212
/// A caller-specified frame ID. Frame IDs must be consistent between
1313
/// subsequent uses of the same data stream.
1414
pub frame_id: u32,
15+
16+
pub composition_time: Option<i32>,
1517
}
1618

1719
impl<'a> EncodedFrame<'a> {

0 commit comments

Comments
 (0)