From 37a3d0a19cf968dc075647ca9c83baee18f65ea7 Mon Sep 17 00:00:00 2001 From: pkeroulas Date: Tue, 25 Aug 2026 15:53:04 -0400 Subject: [PATCH] ffmpeg_plugin: add interlaced ST2110-20 for muxer and demuxer --- ecosystem/ffmpeg_plugin/README.md | 10 ++++ ecosystem/ffmpeg_plugin/mtl_common.c | 38 ++++++++++++ ecosystem/ffmpeg_plugin/mtl_common.h | 4 ++ ecosystem/ffmpeg_plugin/mtl_st20p_rx.c | 83 ++++++++++++++++++++------ ecosystem/ffmpeg_plugin/mtl_st20p_tx.c | 61 +++++++++++++++---- 5 files changed, 169 insertions(+), 27 deletions(-) diff --git a/ecosystem/ffmpeg_plugin/README.md b/ecosystem/ffmpeg_plugin/README.md index ee5722711..7889f9e11 100644 --- a/ecosystem/ffmpeg_plugin/README.md +++ b/ecosystem/ffmpeg_plugin/README.md @@ -144,6 +144,16 @@ formats. The same `-pix_fmt` must be used on TX and RX. | `gbrp12le` | RGB 4:4:4 12-bit RFC 4175 PG2BE12 | yes (Table 1) | | `yuv420p` | Planar I420 passthrough (`ST_FRAME_FMT_YUV420CUSTOM8`) | **no** — MTL-to-MTL only, not interoperable with third-party receivers | +### 2.4. Interlaced video + +Add `-interlaced 1` to either the `mtl_st20p` demuxer or muxer to transport interlaced video. Each ST 2110-20 "frame" then carries a single field, while FFmpeg keeps working on full-height frames whose lines alternate between the first and second field. `-video_size` and `-fps` always describe the full frame, so the wire field rate is twice the configured `-fps`. + +RX: + +```bash +ffmpeg -p_port 0000:af:01.0 -p_sip 192.168.96.2 -p_rx_ip 239.168.85.20 -udp_port 20000 -payload_type 96 -fps 25 -interlaced 1 -pix_fmt yuv422p10le -video_size 1920x1080 -f mtl_st20p -i "k" -f rawvideo /dev/null -y +``` + ## 3. ST22 compressed video run guide A typical workflow for processing an MTL ST22 compressed stream with FFmpeg is outlined in the following steps: Initially, FFmpeg reads a YUV frame from the input source, then forwards the frame to a codec to encode the raw video into a compressed codec stream. Finally, the codec stream is sent to the MTL ST22 plugin. diff --git a/ecosystem/ffmpeg_plugin/mtl_common.c b/ecosystem/ffmpeg_plugin/mtl_common.c index 6ba7405b6..fdf0fbcfc 100644 --- a/ecosystem/ffmpeg_plugin/mtl_common.c +++ b/ecosystem/ffmpeg_plugin/mtl_common.c @@ -316,3 +316,41 @@ int mtl_parse_st30_sample_rate(enum st30_sampling* sample_rate, int value) { return AVERROR(EINVAL); } } + +/* An MTL interlaced frame holds one field's lines packed contiguously, while FFmpeg + * carries both fields woven line by line into a full-height buffer. */ +static void mtl_interlaced_copy(struct st_frame* field, uint8_t* frame, bool to_frame) { + uint8_t planes = st_frame_fmt_planes(field->fmt); + uint32_t field_height = st_frame_data_height(field); + size_t plane_offset = 0; + + for (uint8_t plane = 0; plane < planes; plane++) { + size_t linesize = field->linesize[plane]; + uint8_t* woven = frame + plane_offset + (field->second_field ? linesize : 0); + uint8_t* packed = field->addr[plane]; + + for (uint32_t line = 0; line < field_height; line++) { + uint8_t* src = packed + linesize * line; + uint8_t* dst = woven + linesize * line * 2; + if (to_frame) + mtl_memcpy(dst, src, linesize); + else + mtl_memcpy(src, dst, linesize); + } + plane_offset += linesize * field_height * 2; + } +} + +void mtl_interlaced_field_to_frame(struct st_frame* field, uint8_t* frame) { + mtl_interlaced_copy(field, frame, true); +} + +void mtl_interlaced_frame_to_field(const uint8_t* frame, struct st_frame* field) { + mtl_interlaced_copy(field, (uint8_t*)frame, false); +} + +bool mtl_interlaced_fmt_supported(enum st_frame_fmt fmt) { + /* 420 sampling packs two picture lines into one MTL line, so a single field is not + * line addressable */ + return fmt != ST_FRAME_FMT_YUV420CUSTOM8 && fmt != ST_FRAME_FMT_YUV420PLANAR8; +} diff --git a/ecosystem/ffmpeg_plugin/mtl_common.h b/ecosystem/ffmpeg_plugin/mtl_common.h index fdd05b386..8eac4b7e0 100644 --- a/ecosystem/ffmpeg_plugin/mtl_common.h +++ b/ecosystem/ffmpeg_plugin/mtl_common.h @@ -427,3 +427,7 @@ int mtl_parse_rx_port(AVFormatContext* ctx, const struct StDevArgs* devArgs, int mtl_parse_tx_port(AVFormatContext* ctx, const struct StDevArgs* devArgs, const StTxSessionPortArgs* args, struct st_tx_port* port); int mtl_parse_st30_sample_rate(enum st30_sampling* sample_rate, int value); + +void mtl_interlaced_field_to_frame(struct st_frame* field, uint8_t* frame); +void mtl_interlaced_frame_to_field(const uint8_t* frame, struct st_frame* field); +bool mtl_interlaced_fmt_supported(enum st_frame_fmt fmt); diff --git a/ecosystem/ffmpeg_plugin/mtl_st20p_rx.c b/ecosystem/ffmpeg_plugin/mtl_st20p_rx.c index 8eefdb77d..65b9491a6 100644 --- a/ecosystem/ffmpeg_plugin/mtl_st20p_rx.c +++ b/ecosystem/ffmpeg_plugin/mtl_st20p_rx.c @@ -34,6 +34,7 @@ typedef struct MtlSt20pDemuxerContext { int width, height; enum AVPixelFormat pixel_format; AVRational framerate; + int interlaced; int fb_cnt; int timeout_sec; int session_init_retry; @@ -42,6 +43,7 @@ typedef struct MtlSt20pDemuxerContext { st20p_rx_handle rx_handle; int64_t frame_counter; + int field_size; #ifdef MTL_GPU_DIRECT_ENABLED bool gpu_direct_enabled; @@ -107,6 +109,11 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) { return AVERROR(EINVAL); } ops_rx.height = s->height; + ops_rx.interlaced = s->interlaced ? true : false; + if (ops_rx.interlaced && (s->height % 2)) { + err(ctx, "%s, interlaced needs an even height, got %d\n", __func__, s->height); + return AVERROR(EINVAL); + } ops_rx.fps = framerate_to_st_fps(s->framerate); if (ops_rx.fps == ST_FPS_MAX) { err(ctx, "%s, frame rate %0.2f is not supported\n", __func__, av_q2d(s->framerate)); @@ -165,6 +172,11 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) { err(ctx, "%s, unsupported pixel format: %s\n", __func__, pix_fmt_desc->name); return AVERROR(EINVAL); } + if (ops_rx.interlaced && !mtl_interlaced_fmt_supported(ops_rx.output_fmt)) { + err(ctx, "%s, interlaced is not supported for pixel format: %s\n", __func__, + pix_fmt_desc->name); + return AVERROR(EINVAL); + } img_buf_size = av_image_get_buffer_size(pix_fmt, s->width, s->height, 1); if (img_buf_size < 0) { err(ctx, "%s, av_image_get_buffer_size failed with %d\n", __func__, img_buf_size); @@ -190,6 +202,7 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) { st->codecpar->format = pix_fmt; st->codecpar->width = s->width; st->codecpar->height = s->height; + if (s->interlaced) st->codecpar->field_order = AV_FIELD_TT; avpriv_set_pts_info(st, 64, s->framerate.den, s->framerate.num); ctx->packet_size = img_buf_size; st->codecpar->bit_rate = @@ -238,8 +251,9 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) { st20p_rx_set_block_timeout(s->rx_handle, s->timeout_sec * (uint64_t)NS_PER_S); img_buf_size = st20p_rx_frame_size(s->rx_handle); - if (img_buf_size != ctx->packet_size) { - err(ctx, "%s, frame size mismatch %d:%u\n", __func__, img_buf_size, ctx->packet_size); + s->field_size = s->interlaced ? ctx->packet_size / 2 : ctx->packet_size; + if (img_buf_size != s->field_size) { + err(ctx, "%s, frame size mismatch %d:%d\n", __func__, img_buf_size, s->field_size); mtl_st20p_read_close(ctx); return AVERROR(EIO); } @@ -255,12 +269,9 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) { return 0; } -static int mtl_st20p_read_packet(AVFormatContext* ctx, AVPacket* pkt) { - MtlSt20pDemuxerContext* s = ctx->priv_data; - int ret = 0; - struct st_frame* frame; - - dbg("%s(%d), start\n", __func__, s->idx); +static struct st_frame* mtl_st20p_rx_next_frame(AVFormatContext* ctx, + MtlSt20pDemuxerContext* s) { + struct st_frame* frame = NULL; if (0 == s->frame_counter) { /* @@ -272,19 +283,38 @@ static int mtl_st20p_read_packet(AVFormatContext* ctx, AVPacket* pkt) { if (frame) break; info(ctx, "%s(%d) session initialization retry %d\n", __func__, s->idx, i); } - } else - frame = st20p_rx_get_frame(s->rx_handle); + } + if (!frame) frame = st20p_rx_get_frame(s->rx_handle); if (!frame) { info(ctx, "%s(%d), st20p_rx_get_frame timeout\n", __func__, s->idx); - return AVERROR(EIO); + return NULL; } dbg(ctx, "%s(%d), st20p_rx_get_frame: %p\n", __func__, s->idx, frame); - if (frame->data_size != ctx->packet_size) { - err(ctx, "%s(%d), unexpected frame size received: %" PRId64 " (%u expected)\n", - __func__, s->idx, frame->data_size, ctx->packet_size); + if (frame->data_size != s->field_size) { + err(ctx, "%s(%d), unexpected frame size received: %" PRId64 " (%d expected)\n", + __func__, s->idx, frame->data_size, s->field_size); st20p_rx_put_frame(s->rx_handle, frame); - return AVERROR(EIO); + return NULL; + } + return frame; +} + +static int mtl_st20p_read_packet(AVFormatContext* ctx, AVPacket* pkt) { + MtlSt20pDemuxerContext* s = ctx->priv_data; + int ret = 0; + struct st_frame* frame; + + dbg("%s(%d), start\n", __func__, s->idx); + + frame = mtl_st20p_rx_next_frame(ctx, s); + if (!frame) return AVERROR(EIO); + + /* a packet always starts on a first field, drop fields until the parity matches */ + while (s->interlaced && frame->second_field) { + st20p_rx_put_frame(s->rx_handle, frame); + frame = mtl_st20p_rx_next_frame(ctx, s); + if (!frame) return AVERROR(EIO); } ret = av_new_packet(pkt, ctx->packet_size); @@ -294,8 +324,19 @@ static int mtl_st20p_read_packet(AVFormatContext* ctx, AVPacket* pkt) { return ret; } - /* todo: zero copy with external frame mode */ - mtl_memcpy(pkt->data, frame->addr[0], ctx->packet_size); + if (s->interlaced) { + mtl_interlaced_field_to_frame(frame, pkt->data); + st20p_rx_put_frame(s->rx_handle, frame); + frame = mtl_st20p_rx_next_frame(ctx, s); + if (!frame) { + av_packet_unref(pkt); + return AVERROR(EIO); + } + mtl_interlaced_field_to_frame(frame, pkt->data); + } else { + /* todo: zero copy with external frame mode */ + mtl_memcpy(pkt->data, frame->addr[0], ctx->packet_size); + } st20p_rx_put_frame(s->rx_handle, frame); pkt->pts = pkt->dts = s->frame_counter++; @@ -344,6 +385,14 @@ static const AVOption mtl_st20p_rx_options[] = { 0, 1000, DEC}, + {"interlaced", + "Interlaced video, each frame carries two ST 2110-20 fields", + OFFSET(interlaced), + AV_OPT_TYPE_BOOL, + {.i64 = 0}, + 0, + 1, + DEC}, {"timeout_s", "Frame get timeout in seconds", OFFSET(timeout_sec), diff --git a/ecosystem/ffmpeg_plugin/mtl_st20p_tx.c b/ecosystem/ffmpeg_plugin/mtl_st20p_tx.c index a29b8a4e8..103000a3c 100644 --- a/ecosystem/ffmpeg_plugin/mtl_st20p_tx.c +++ b/ecosystem/ffmpeg_plugin/mtl_st20p_tx.c @@ -31,6 +31,7 @@ typedef struct mtlSt20pMuxerContext { int fb_cnt; int width; int height; + int interlaced; enum AVPixelFormat pixel_format; AVRational framerate; mtl_handle dev_handle; @@ -38,6 +39,7 @@ typedef struct mtlSt20pMuxerContext { int64_t frame_counter; int frame_size; + int packet_size; } mtlSt20pMuxerContext; static int mtl_st20p_write_close(AVFormatContext* ctx) { @@ -79,6 +81,11 @@ static int mtl_st20p_write_header(AVFormatContext* ctx) { ops_tx.width = s->width = ctx->streams[0]->codecpar->width; ops_tx.height = s->height = ctx->streams[0]->codecpar->height; + ops_tx.interlaced = s->interlaced ? true : false; + if (ops_tx.interlaced && (s->height % 2)) { + err(ctx, "%s, interlaced needs an even height, got %d\n", __func__, s->height); + return AVERROR(EINVAL); + } s->framerate = ctx->streams[0]->avg_frame_rate; ops_tx.fps = framerate_to_st_fps(s->framerate); if (ops_tx.fps == ST_FPS_MAX) { @@ -141,6 +148,11 @@ static int mtl_st20p_write_header(AVFormatContext* ctx) { err(ctx, "%s, unsupported pixel format: %d\n", __func__, s->pixel_format); return AVERROR(EINVAL); } + if (ops_tx.interlaced && !mtl_interlaced_fmt_supported(ops_tx.input_fmt)) { + err(ctx, "%s, interlaced is not supported for pixel format: %d\n", __func__, + s->pixel_format); + return AVERROR(EINVAL); + } ops_tx.name = "st20p_ffmpge"; ops_tx.priv = s; // Handle of priv_data registered to lib @@ -170,21 +182,16 @@ static int mtl_st20p_write_header(AVFormatContext* ctx) { } s->frame_size = st20p_tx_frame_size(s->tx_handle); + s->packet_size = s->interlaced ? s->frame_size * 2 : s->frame_size; info(ctx, "%s(%d), tx_handle %p\n", __func__, s->idx, s->tx_handle); return 0; } -static int mtl_st20p_write_packet(AVFormatContext* ctx, AVPacket* pkt) { +static int mtl_st20p_write_field(AVFormatContext* ctx, const AVPacket* pkt, + bool second_field) { mtlSt20pMuxerContext* s = ctx->priv_data; struct st_frame* frame; - if (pkt->size != s->frame_size) { - err(ctx, "%s(%d), unexpected pkt size: %d (%d expected)\n", __func__, s->idx, - pkt->size, s->frame_size); - return AVERROR(EIO); - } - - dbg("%s(%d), start\n", __func__, s->idx); frame = st20p_tx_get_frame(s->tx_handle); if (!frame) { info(ctx, "%s(%d), st20p_tx_get_frame timeout\n", __func__, s->idx); @@ -192,10 +199,36 @@ static int mtl_st20p_write_packet(AVFormatContext* ctx, AVPacket* pkt) { } dbg(ctx, "%s(%d), st20p_tx_get_frame: %p\n", __func__, s->idx, frame); - /* todo: zero copy with external frame mode */ - mtl_memcpy(frame->addr[0], pkt->data, s->frame_size); + if (s->interlaced) { + frame->second_field = second_field; + mtl_interlaced_frame_to_field(pkt->data, frame); + } else { + /* todo: zero copy with external frame mode */ + mtl_memcpy(frame->addr[0], pkt->data, s->frame_size); + } st20p_tx_put_frame(s->tx_handle, frame); + return 0; +} + +static int mtl_st20p_write_packet(AVFormatContext* ctx, AVPacket* pkt) { + mtlSt20pMuxerContext* s = ctx->priv_data; + int ret; + + if (pkt->size != s->packet_size) { + err(ctx, "%s(%d), unexpected pkt size: %d (%d expected)\n", __func__, s->idx, + pkt->size, s->packet_size); + return AVERROR(EIO); + } + + dbg("%s(%d), start\n", __func__, s->idx); + ret = mtl_st20p_write_field(ctx, pkt, false); + if (ret < 0) return ret; + if (s->interlaced) { + ret = mtl_st20p_write_field(ctx, pkt, true); + if (ret < 0) return ret; + } + s->frame_counter++; dbg(ctx, "%s(%d), frame counter %" PRId64 "\n", __func__, s->idx, s->frame_counter); return 0; @@ -216,6 +249,14 @@ static const AVOption mtl_st20p_tx_options[] = { 3, 8, ENC}, + {"interlaced", + "Interlaced video, each frame carries two ST 2110-20 fields", + OFFSET(interlaced), + AV_OPT_TYPE_BOOL, + {.i64 = 0}, + 0, + 1, + ENC}, {NULL}, };