Make NV12 pixel format work

Works for preview (all types), JPEG capture, video recording.

Signed-off-by: David Plowman <david.plowman@raspberrypi.com>

Author: davidplowman

picamera2/encoders/libav_h264_encoder.py

@@ -121,7 +121,14 @@ def _start(self):
         self._stream.codec_context.time_base = Fraction(1, 1000000)
         self._stream.codec_context.options["tune"] = "zerolatency"
 
-        FORMAT_TABLE = {"YUV420": "yuv420p", "BGR888": "rgb24", "RGB888": "bgr24", "XBGR8888": "rgba", "XRGB8888": "bgra"}
+        FORMAT_TABLE = {
+            "YUV420": "yuv420p",
+            "BGR888": "rgb24",
+            "RGB888": "bgr24",
+            "XBGR8888": "rgba",
+            "XRGB8888": "bgra",
+            "NV12": "nv12",
+        }
         self._av_input_format = FORMAT_TABLE[self._format]
 
         self._request_release_queue = collections.deque()

picamera2/previews/drm_preview.py

@@ -57,6 +57,7 @@ class DrmPreview(NullPreview):
         "XBGR8888": PixelFormat.XBGR8888,
         "YUV420": PixelFormat.YUV420,
         "YVU420": PixelFormat.YVU420,
+        "NV12": PixelFormat.NV12,
         "MJPEG": PixelFormat.BGR888,
     }
 
@@ -223,6 +224,9 @@ def render_drm(self, picam2, completed_request):
                     drmfb = pykms.DmabufFramebuffer(
                         self.card, width, height, fmt, [fd, fd, fd], [stride, stride2, stride2], [0, size, size + h2 * stride2]
                     )
+                elif pixel_format == "NV12":
+                    size = height * stride
+                    drmfb = pykms.DmabufFramebuffer(self.card, width, height, fmt, [fd, fd], [stride, stride], [0, size])
                 else:
                     drmfb = pykms.DmabufFramebuffer(self.card, width, height, fmt, [fd], [stride], [0])
                 self.drmfbs[fb] = drmfb

picamera2/previews/q_gl_picamera2.py

@@ -271,6 +271,7 @@ class Buffer:
                 # doesn't work "VYUY": "VYUY",
                 "YUV420": "YU12",
                 "YVU420": "YV12",
+                "NV12": "NV12",
             }
 
             def __init__(self, display, completed_request, max_texture_size):
@@ -306,6 +307,22 @@ def __init__(self, display, completed_request, max_texture_size):
                         EGL_NONE,
                     ]
                     # fmt: on
+                elif pixel_format == "NV12":
+                    h2 = h // 2
+                    # fmt: off
+                    attribs = [
+                        EGL_WIDTH, w,
+                        EGL_HEIGHT, h,
+                        EGL_LINUX_DRM_FOURCC_EXT, fmt,
+                        EGL_DMA_BUF_PLANE0_FD_EXT, fb.planes[0].fd,
+                        EGL_DMA_BUF_PLANE0_OFFSET_EXT, 0,
+                        EGL_DMA_BUF_PLANE0_PITCH_EXT, cfg.stride,
+                        EGL_DMA_BUF_PLANE1_FD_EXT, fb.planes[0].fd,
+                        EGL_DMA_BUF_PLANE1_OFFSET_EXT, h * cfg.stride,
+                        EGL_DMA_BUF_PLANE1_PITCH_EXT, cfg.stride,
+                        EGL_NONE,
+                    ]
+                    # fmt: on
                 else:
                     # fmt: off
                     attribs = [

picamera2/previews/q_picamera2.py

@@ -199,12 +199,11 @@ def render_request(self, completed_request):
             stream_config = camera_config[display_stream_name]
 
             img = completed_request.make_array(display_stream_name)
-            if stream_config["format"] in ("YUV420", "YUYV"):
+            fmt = stream_config["format"]
+            if fmt in ("YUV420", "YUYV", "NV12"):
                 if cv2_available:
-                    if stream_config["format"] == "YUV420":
-                        img = cv2.cvtColor(img, cv2.COLOR_YUV420p2BGR)
-                    else:
-                        img = cv2.cvtColor(img, cv2.COLOR_YUV2RGB_YUYV)
+                    table = {"YUV420": cv2.COLOR_YUV420p2BGR, "YUYV": cv2.COLOR_YUV2RGB_YUYV, "NV12": cv2.COLOR_YUV2RGB_NV12}
+                    img = cv2.cvtColor(img, table[fmt])
                 else:
                     logging.error("Qt preview cannot display YUV420/YUYV without cv2")
                     return

picamera2/request.py

@@ -244,6 +244,13 @@ def save(
                     V = reshaped[2 * height + height // 2 :, : width // 2]
                     output_bytes = simplejpeg.encode_jpeg_yuv_planes(Y, U, V, quality)
                     Y = reshaped = U = V = None
+                elif format == 'NV12':
+                    width, height = self.config[name]['size']
+                    Y = m.array[:height, :width]
+                    U = np.ascontiguousarray(m.array[height : height + height // 2, 0 : 2 * width : 2])
+                    V = np.ascontiguousarray(m.array[height : height + height // 2, 1 : 2 * width : 2])
+                    output_bytes = simplejpeg.encode_jpeg_yuv_planes(Y, U, V, quality)
+                    Y = U = V = None
                 else:
                     FORMAT_TABLE = {"XBGR8888": "RGBX", "XRGB8888": "BGRX", "BGR888": "RGB", "RGB888": "BGR"}
                     output_bytes = simplejpeg.encode_jpeg(m.array, quality, FORMAT_TABLE[format], '420')
@@ -326,6 +333,10 @@ def _make_array_shared(self, buffer: np.ndarray, config: Dict[str, Any]) -> np.n
             # These dimensions seem a bit strange, but mean that
             # cv2.cvtColor(image, cv2.COLOR_YUV2BGR_YUYV) will convert directly to RGB.
             image = array.reshape(h, stride // 2, 2)
+        elif fmt == "NV12":
+            # The UV rows should be as long as the Y rows, so we can slice off any
+            # padding.
+            image = array.reshape((h * 3 // 2, stride))[:w]
         elif fmt == "MJPEG":
             image = np.array(Image.open(io.BytesIO(array)))  # type: ignore
         elif formats.is_raw(fmt):

tests/nv12.py

@@ -0,0 +1,38 @@
+#!/usr/bin/python3
+
+# Check that the "NV12" pixel format works.
+
+import time
+
+from picamera2 import Picamera2, Preview
+from picamera2.encoders import H264Encoder
+from picamera2.outputs import PyavOutput
+
+picam2 = Picamera2()
+config = picam2.create_video_configuration({'format': 'NV12', 'size': (640, 360)})
+picam2.configure(config)
+
+picam2.start_preview(Preview.QTGL)
+picam2.start()
+
+time.sleep(1)
+
+picam2.stop_preview()
+picam2.start_preview(Preview.QT)
+
+time.sleep(1)
+
+picam2.stop_preview()
+picam2.start_preview()
+
+time.sleep(1)
+
+picam2.capture_file("check.jpg")
+
+encoder = H264Encoder(bitrate=5000000)
+output = PyavOutput("check.mp4")
+picam2.start_recording(encoder, output)
+
+time.sleep(5)
+
+picam2.stop_recording()

tests/test_list.txt

@@ -95,6 +95,7 @@ tests/null_encoder.py
 tests/mode_test.py
 tests/multicamera.py
 tests/multicamera_2.py
+tests/nv12.py
 tests/preview_cycle_test.py
 tests/preview_location_test.py
 tests/preview_start_stop.py