ffmpeg_plugin: add support for interlaced ST2110-20 muxer and demuxer

ecosystem/ffmpeg_plugin/README.md

@@ -144,6 +144,16 @@ formats. The same `-pix_fmt` must be used on TX and RX.
 | `gbrp12le`    | RGB 4:4:4 12-bit RFC 4175 PG2BE12 | yes (Table 1)              |
 | `yuv420p`     | Planar I420 passthrough (`ST_FRAME_FMT_YUV420CUSTOM8`) | **no** — MTL-to-MTL only, not interoperable with third-party receivers |
 
+### 2.4. Interlaced video
+
+Add `-interlaced 1` to either the `mtl_st20p` demuxer or muxer to transport interlaced video. Each ST 2110-20 "frame" then carries a single field, while FFmpeg keeps working on full-height frames whose lines alternate between the first and second field. `-video_size` and `-fps` always describe the full frame, so the wire field rate is twice the configured `-fps`.
+
+RX:
+
+```bash
+ffmpeg -p_port 0000:af:01.0 -p_sip 192.168.96.2 -p_rx_ip 239.168.85.20 -udp_port 20000 -payload_type 96 -fps 25 -interlaced 1 -pix_fmt yuv422p10le -video_size 1920x1080 -f mtl_st20p -i "k" -f rawvideo /dev/null -y
+```
+
 ## 3. ST22 compressed video run guide
 
 A typical workflow for processing an MTL ST22 compressed stream with FFmpeg is outlined in the following steps: Initially, FFmpeg reads a YUV frame from the input source, then forwards the frame to a codec to encode the raw video into a compressed codec stream. Finally, the codec stream is sent to the MTL ST22 plugin.

ecosystem/ffmpeg_plugin/mtl_common.c

@@ -316,3 +316,41 @@ int mtl_parse_st30_sample_rate(enum st30_sampling* sample_rate, int value) {
       return AVERROR(EINVAL);
   }
 }
+
+/* An MTL interlaced frame holds one field's lines packed contiguously, while FFmpeg
+ * carries both fields woven line by line into a full-height buffer. */
+static void mtl_interlaced_copy(struct st_frame* field, uint8_t* frame, bool to_frame) {
+  uint8_t planes = st_frame_fmt_planes(field->fmt);
+  uint32_t field_height = st_frame_data_height(field);
+  size_t plane_offset = 0;
+
+  for (uint8_t plane = 0; plane < planes; plane++) {
+    size_t linesize = field->linesize[plane];
+    uint8_t* woven = frame + plane_offset + (field->second_field ? linesize : 0);
+    uint8_t* packed = field->addr[plane];
+
+    for (uint32_t line = 0; line < field_height; line++) {
+      uint8_t* src = packed + linesize * line;
+      uint8_t* dst = woven + linesize * line * 2;
+      if (to_frame)
+        mtl_memcpy(dst, src, linesize);
+      else
+        mtl_memcpy(src, dst, linesize);
+    }
+    plane_offset += linesize * field_height * 2;
+  }
+}
+
+void mtl_interlaced_field_to_frame(struct st_frame* field, uint8_t* frame) {
+  mtl_interlaced_copy(field, frame, true);
+}
+
+void mtl_interlaced_frame_to_field(const uint8_t* frame, struct st_frame* field) {
+  mtl_interlaced_copy(field, (uint8_t*)frame, false);
+}
+
+bool mtl_interlaced_fmt_supported(enum st_frame_fmt fmt) {
+  /* 420 sampling packs two picture lines into one MTL line, so a single field is not
+   * line addressable */
+  return fmt != ST_FRAME_FMT_YUV420CUSTOM8 && fmt != ST_FRAME_FMT_YUV420PLANAR8;
+}

ecosystem/ffmpeg_plugin/mtl_common.h

@@ -427,3 +427,7 @@ int mtl_parse_rx_port(AVFormatContext* ctx, const struct StDevArgs* devArgs,
 int mtl_parse_tx_port(AVFormatContext* ctx, const struct StDevArgs* devArgs,
                       const StTxSessionPortArgs* args, struct st_tx_port* port);
 int mtl_parse_st30_sample_rate(enum st30_sampling* sample_rate, int value);
+
+void mtl_interlaced_field_to_frame(struct st_frame* field, uint8_t* frame);
+void mtl_interlaced_frame_to_field(const uint8_t* frame, struct st_frame* field);
+bool mtl_interlaced_fmt_supported(enum st_frame_fmt fmt);

ecosystem/ffmpeg_plugin/mtl_st20p_rx.c

@@ -34,6 +34,7 @@ typedef struct MtlSt20pDemuxerContext {
   int width, height;
   enum AVPixelFormat pixel_format;
   AVRational framerate;
+  int interlaced;
   int fb_cnt;
   int timeout_sec;
   int session_init_retry;
@@ -42,6 +43,7 @@ typedef struct MtlSt20pDemuxerContext {
   st20p_rx_handle rx_handle;
 
   int64_t frame_counter;
+  int field_size;
 
 #ifdef MTL_GPU_DIRECT_ENABLED
   bool gpu_direct_enabled;
@@ -107,6 +109,11 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) {
     return AVERROR(EINVAL);
   }
   ops_rx.height = s->height;
+  ops_rx.interlaced = s->interlaced ? true : false;
+  if (ops_rx.interlaced && (s->height % 2)) {
+    err(ctx, "%s, interlaced needs an even height, got %d\n", __func__, s->height);
+    return AVERROR(EINVAL);
+  }
   ops_rx.fps = framerate_to_st_fps(s->framerate);
   if (ops_rx.fps == ST_FPS_MAX) {
     err(ctx, "%s, frame rate %0.2f is not supported\n", __func__, av_q2d(s->framerate));
@@ -165,6 +172,11 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) {
       err(ctx, "%s, unsupported pixel format: %s\n", __func__, pix_fmt_desc->name);
       return AVERROR(EINVAL);
   }
+  if (ops_rx.interlaced && !mtl_interlaced_fmt_supported(ops_rx.output_fmt)) {
+    err(ctx, "%s, interlaced is not supported for pixel format: %s\n", __func__,
+        pix_fmt_desc->name);
+    return AVERROR(EINVAL);
+  }
   img_buf_size = av_image_get_buffer_size(pix_fmt, s->width, s->height, 1);
   if (img_buf_size < 0) {
     err(ctx, "%s, av_image_get_buffer_size failed with %d\n", __func__, img_buf_size);
@@ -190,6 +202,7 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) {
   st->codecpar->format = pix_fmt;
   st->codecpar->width = s->width;
   st->codecpar->height = s->height;
+  if (s->interlaced) st->codecpar->field_order = AV_FIELD_TT;
   avpriv_set_pts_info(st, 64, s->framerate.den, s->framerate.num);
   ctx->packet_size = img_buf_size;
   st->codecpar->bit_rate =
@@ -238,8 +251,9 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) {
     st20p_rx_set_block_timeout(s->rx_handle, s->timeout_sec * (uint64_t)NS_PER_S);
 
   img_buf_size = st20p_rx_frame_size(s->rx_handle);
-  if (img_buf_size != ctx->packet_size) {
-    err(ctx, "%s, frame size mismatch %d:%u\n", __func__, img_buf_size, ctx->packet_size);
+  s->field_size = s->interlaced ? ctx->packet_size / 2 : ctx->packet_size;
+  if (img_buf_size != s->field_size) {
+    err(ctx, "%s, frame size mismatch %d:%d\n", __func__, img_buf_size, s->field_size);
     mtl_st20p_read_close(ctx);
     return AVERROR(EIO);
   }
@@ -255,12 +269,9 @@ static int mtl_st20p_read_header(AVFormatContext* ctx) {
   return 0;
 }
 
-static int mtl_st20p_read_packet(AVFormatContext* ctx, AVPacket* pkt) {
-  MtlSt20pDemuxerContext* s = ctx->priv_data;
-  int ret = 0;
-  struct st_frame* frame;
-
-  dbg("%s(%d), start\n", __func__, s->idx);
+static struct st_frame* mtl_st20p_rx_next_frame(AVFormatContext* ctx,
+                                                MtlSt20pDemuxerContext* s) {
+  struct st_frame* frame = NULL;
 
   if (0 == s->frame_counter) {
     /*
@@ -272,19 +283,38 @@ static int mtl_st20p_read_packet(AVFormatContext* ctx, AVPacket* pkt) {
       if (frame) break;
       info(ctx, "%s(%d) session initialization retry %d\n", __func__, s->idx, i);
     }
-  } else
-    frame = st20p_rx_get_frame(s->rx_handle);
+  }
+  if (!frame) frame = st20p_rx_get_frame(s->rx_handle);
 
   if (!frame) {
     info(ctx, "%s(%d), st20p_rx_get_frame timeout\n", __func__, s->idx);
-    return AVERROR(EIO);
+    return NULL;
   }
   dbg(ctx, "%s(%d), st20p_rx_get_frame: %p\n", __func__, s->idx, frame);
-  if (frame->data_size != ctx->packet_size) {
-    err(ctx, "%s(%d), unexpected frame size received: %" PRId64 " (%u expected)\n",
-        __func__, s->idx, frame->data_size, ctx->packet_size);
+  if (frame->data_size != s->field_size) {
+    err(ctx, "%s(%d), unexpected frame size received: %" PRId64 " (%d expected)\n",
+        __func__, s->idx, frame->data_size, s->field_size);
     st20p_rx_put_frame(s->rx_handle, frame);
-    return AVERROR(EIO);
+    return NULL;
+  }
+  return frame;
+}
+
+static int mtl_st20p_read_packet(AVFormatContext* ctx, AVPacket* pkt) {
+  MtlSt20pDemuxerContext* s = ctx->priv_data;
+  int ret = 0;
+  struct st_frame* frame;
+
+  dbg("%s(%d), start\n", __func__, s->idx);
+
+  frame = mtl_st20p_rx_next_frame(ctx, s);
+  if (!frame) return AVERROR(EIO);
+
+  /* a packet always starts on a first field, drop fields until the parity matches */
+  while (s->interlaced && frame->second_field) {
+    st20p_rx_put_frame(s->rx_handle, frame);
+    frame = mtl_st20p_rx_next_frame(ctx, s);
+    if (!frame) return AVERROR(EIO);
   }
 
   ret = av_new_packet(pkt, ctx->packet_size);
@@ -294,8 +324,19 @@ static int mtl_st20p_read_packet(AVFormatContext* ctx, AVPacket* pkt) {
     return ret;
   }
 
-  /* todo: zero copy with external frame mode */
-  mtl_memcpy(pkt->data, frame->addr[0], ctx->packet_size);
+  if (s->interlaced) {
+    mtl_interlaced_field_to_frame(frame, pkt->data);
+    st20p_rx_put_frame(s->rx_handle, frame);
+    frame = mtl_st20p_rx_next_frame(ctx, s);
+    if (!frame) {
+      av_packet_unref(pkt);
+      return AVERROR(EIO);
+    }
+    mtl_interlaced_field_to_frame(frame, pkt->data);
+  } else {
+    /* todo: zero copy with external frame mode */
+    mtl_memcpy(pkt->data, frame->addr[0], ctx->packet_size);
+  }
   st20p_rx_put_frame(s->rx_handle, frame);
 
   pkt->pts = pkt->dts = s->frame_counter++;
@@ -344,6 +385,14 @@ static const AVOption mtl_st20p_rx_options[] = {
      0,
      1000,
      DEC},
+    {"interlaced",
+     "Interlaced video, each frame carries two ST 2110-20 fields",
+     OFFSET(interlaced),
+     AV_OPT_TYPE_BOOL,
+     {.i64 = 0},
+     0,
+     1,
+     DEC},
     {"timeout_s",
      "Frame get timeout in seconds",
      OFFSET(timeout_sec),

ecosystem/ffmpeg_plugin/mtl_st20p_tx.c

@@ -31,13 +31,15 @@ typedef struct mtlSt20pMuxerContext {
   int fb_cnt;
   int width;
   int height;
+  int interlaced;
   enum AVPixelFormat pixel_format;
   AVRational framerate;
   mtl_handle dev_handle;
   st20p_tx_handle tx_handle;
 
   int64_t frame_counter;
   int frame_size;
+  int packet_size;
 } mtlSt20pMuxerContext;
 
 static int mtl_st20p_write_close(AVFormatContext* ctx) {
@@ -79,6 +81,11 @@ static int mtl_st20p_write_header(AVFormatContext* ctx) {
 
   ops_tx.width = s->width = ctx->streams[0]->codecpar->width;
   ops_tx.height = s->height = ctx->streams[0]->codecpar->height;
+  ops_tx.interlaced = s->interlaced ? true : false;
+  if (ops_tx.interlaced && (s->height % 2)) {
+    err(ctx, "%s, interlaced needs an even height, got %d\n", __func__, s->height);
+    return AVERROR(EINVAL);
+  }
   s->framerate = ctx->streams[0]->avg_frame_rate;
   ops_tx.fps = framerate_to_st_fps(s->framerate);
   if (ops_tx.fps == ST_FPS_MAX) {
@@ -141,6 +148,11 @@ static int mtl_st20p_write_header(AVFormatContext* ctx) {
       err(ctx, "%s, unsupported pixel format: %d\n", __func__, s->pixel_format);
       return AVERROR(EINVAL);
   }
+  if (ops_tx.interlaced && !mtl_interlaced_fmt_supported(ops_tx.input_fmt)) {
+    err(ctx, "%s, interlaced is not supported for pixel format: %d\n", __func__,
+        s->pixel_format);
+    return AVERROR(EINVAL);
+  }
 
   ops_tx.name = "st20p_ffmpge";
   ops_tx.priv = s;  // Handle of priv_data registered to lib
@@ -170,32 +182,53 @@ static int mtl_st20p_write_header(AVFormatContext* ctx) {
   }
 
   s->frame_size = st20p_tx_frame_size(s->tx_handle);
+  s->packet_size = s->interlaced ? s->frame_size * 2 : s->frame_size;
   info(ctx, "%s(%d), tx_handle %p\n", __func__, s->idx, s->tx_handle);
   return 0;
 }
 
-static int mtl_st20p_write_packet(AVFormatContext* ctx, AVPacket* pkt) {
+static int mtl_st20p_write_field(AVFormatContext* ctx, const AVPacket* pkt,
+                                 bool second_field) {
   mtlSt20pMuxerContext* s = ctx->priv_data;
   struct st_frame* frame;
 
-  if (pkt->size != s->frame_size) {
-    err(ctx, "%s(%d), unexpected pkt size: %d (%d expected)\n", __func__, s->idx,
-        pkt->size, s->frame_size);
-    return AVERROR(EIO);
-  }
-
-  dbg("%s(%d), start\n", __func__, s->idx);
   frame = st20p_tx_get_frame(s->tx_handle);
   if (!frame) {
     info(ctx, "%s(%d), st20p_tx_get_frame timeout\n", __func__, s->idx);
     return AVERROR(EIO);
   }
   dbg(ctx, "%s(%d), st20p_tx_get_frame: %p\n", __func__, s->idx, frame);
 
-  /* todo: zero copy with external frame mode */
-  mtl_memcpy(frame->addr[0], pkt->data, s->frame_size);
+  if (s->interlaced) {
+    frame->second_field = second_field;
+    mtl_interlaced_frame_to_field(pkt->data, frame);
+  } else {
+    /* todo: zero copy with external frame mode */
+    mtl_memcpy(frame->addr[0], pkt->data, s->frame_size);
+  }
 
   st20p_tx_put_frame(s->tx_handle, frame);
+  return 0;
+}
+
+static int mtl_st20p_write_packet(AVFormatContext* ctx, AVPacket* pkt) {
+  mtlSt20pMuxerContext* s = ctx->priv_data;
+  int ret;
+
+  if (pkt->size != s->packet_size) {
+    err(ctx, "%s(%d), unexpected pkt size: %d (%d expected)\n", __func__, s->idx,
+        pkt->size, s->packet_size);
+    return AVERROR(EIO);
+  }
+
+  dbg("%s(%d), start\n", __func__, s->idx);
+  ret = mtl_st20p_write_field(ctx, pkt, false);
+  if (ret < 0) return ret;
+  if (s->interlaced) {
+    ret = mtl_st20p_write_field(ctx, pkt, true);
+    if (ret < 0) return ret;
+  }
+
   s->frame_counter++;
   dbg(ctx, "%s(%d), frame counter %" PRId64 "\n", __func__, s->idx, s->frame_counter);
   return 0;
@@ -216,6 +249,14 @@ static const AVOption mtl_st20p_tx_options[] = {
      3,
      8,
      ENC},
+    {"interlaced",
+     "Interlaced video, each frame carries two ST 2110-20 fields",
+     OFFSET(interlaced),
+     AV_OPT_TYPE_BOOL,
+     {.i64 = 0},
+     0,
+     1,
+     ENC},
     {NULL},
 };