Update and improve video capture sample

samples/subsys/video/capture/README.rst

@@ -7,25 +7,33 @@
 Description
 ***********
 
-This sample application uses the :ref:`Video API <video_api>` to retrieve video frames from the
-video capture device, writes a frame count message to the console, and then
-discards the video frame data.
+This sample application uses the :ref:`video_api` to capture frames from a video capture
+device then uses the :ref:`display_api` to display them onto an LCD screen (if any).
 
 Requirements
 ************
 
-This sample requires a video capture device (e.g. a camera).
+This sample needs a video capture device (e.g. a camera) but it is not mandatory.
+Supported camera modules on some i.MX RT boards can be found below.
+
+- `Camera iMXRT`_
 
 - :ref:`mimxrt1064_evk`
 - `MT9M114 camera module`_
 
+- :ref:`mimxrt1170_evk`
+- `OV5640 camera module`_
+
 Wiring
 ******
 
-On :ref:`mimxrt1064_evk`, The MT9M114 camera module should be plugged in the
+On :ref:`mimxrt1064_evk`, the MT9M114 camera module should be plugged in the
 J35 camera connector. A USB cable should be connected from a host to the micro
-USB debug connector (J41) in order to get console output via the freelink
-interface.
+USB debug connector (J41) in order to get console output via the freelink interface.
+
+On :ref:`mimxrt1170_evk`, the OV5640 camera module should be plugged into the
+J2 camera connector. A USB cable should be connected from a host to the micro
+USB debug connector (J11) in order to get console output via the daplink interface.
 
 Building and Running
 ********************
@@ -35,30 +43,57 @@ For :ref:`mimxrt1064_evk`, build this sample application with the following comm
 .. zephyr-app-commands::
    :zephyr-app: samples/subsys/video/capture
    :board: mimxrt1064_evk
+   :shield: "dvp_fpc24_mt9m114 rk043fn66hs_ctg"
    :goals: build
    :compact:
 
+For :ref:`mimxrt1170_evk`, build this sample application with the following commands:
+
+.. zephyr-app-commands::
+   :zephyr-app: samples/subsys/video/capture
+   :board: mimxrt1170_evk/mimxrt1176/cm7
+   :shield: "nxp_btb44_ov5640 rk055hdmipi4ma0"
+   :goals: build
+   :compact:
+
+For testing purpose without the need of any real video capture and/or display hardwares,
+a video software pattern generator is supported by the above build commands without
+specifying the shields.
+
 Sample Output
 =============
 
 .. code-block:: console
 
-    Found video device: CSI
-    width (640,640), height (480,480)
-    Supported pixelformats (fourcc):
-     - RGBP
-    Use default format (640x480)
+    Video device: csi@402bc000
+    - Capabilities:
+      RGBP width [480; 480; 0] height [272; 272; 0]
+      YUYV width [480; 480; 0] height [272; 272; 0]
+      RGBP width [640; 640; 0] height [480; 480; 0]
+      YUYV width [640; 640; 0] height [480; 480; 0]
+      RGBP width [1280; 1280; 0] height [720; 720; 0]
+      YUYV width [1280; 1280; 0] height [720; 720; 0]
+    - Default format: RGBP 480x272
+
+    Display device: display-controller@402b8000
+    - Capabilities:
+      x_resolution = 480, y_resolution = 272, supported_pixel_formats = 40
+      current_pixel_format = 32, current_orientation = 0
+
     Capture started
-    Got frame 743! size: 614400; timestamp 100740 ms
-    Got frame 744! size: 614400; timestamp 100875 ms
-    Got frame 745! size: 614400; timestamp 101010 ms
-    Got frame 746! size: 614400; timestamp 101146 ms
-    Got frame 747! size: 614400; timestamp 101281 ms
-    Got frame 748! size: 614400; timestamp 101416 ms
+    Got frame 0! size: 261120; timestamp 249 ms
+    Got frame 1! size: 261120; timestamp 282 ms
+    Got frame 2! size: 261120; timestamp 316 ms
+    Got frame 3! size: 261120; timestamp 350 ms
+    Got frame 4! size: 261120; timestamp 384 ms
+    Got frame 5! size: 261120; timestamp 418 ms
+    Got frame 6! size: 261120; timestamp 451 ms
 
    <repeats endlessly>
 
 References
 **********
 
+.. _Camera iMXRT: https://community.nxp.com/t5/i-MX-RT-Knowledge-Base/Connecting-camera-and-LCD-to-i-MX-RT-EVKs/ta-p/1122183
 .. _MT9M114 camera module: https://www.onsemi.com/PowerSolutions/product.do?id=MT9M114
+.. _OV5640 camera module: https://cdn.sparkfun.com/datasheets/Sensors/LightImaging/OV5640_datasheet.pdf

samples/subsys/video/capture/boards/mimxrt1170_evk_mimxrt1176_cm7.conf

@@ -0,0 +1,4 @@
+CONFIG_VIDEO_BUFFER_POOL_SZ_MAX=3686800
+CONFIG_DMA=y
+CONFIG_MCUX_ELCDIF_PXP=y
+CONFIG_MCUX_ELCDIF_PXP_ROTATE_90=y

samples/subsys/video/capture/prj.conf

@@ -4,3 +4,4 @@ CONFIG_SHELL=y
 CONFIG_DEVICE_SHELL=y
 CONFIG_PRINTK=y
 CONFIG_LOG=y
+CONFIG_DISPLAY=y

samples/subsys/video/capture/sample.yaml

@@ -3,10 +3,18 @@ sample:
 tests:
   sample.video.capture:
     build_only: true
-    tags: video
+    tags:
+      - video
+      - shield
+      - samples
+    extra_args:
+      - platform:mimxrt1064_evk:SHIELD="dvp_fpc24_mt9m114;rk043fn66hs_ctg"
+      - platform:mimxrt1170_evk/mimxrt1176/cm7:SHIELD="nxp_btb44_ov5640;rk055hdmipi4ma0"
     platform_allow:
       - mimxrt1064_evk
+      - mimxrt1170_evk/mimxrt1176/cm7
       - mm_swiftio
     depends_on: video
     integration_platforms:
       - mimxrt1064_evk
+      - mimxrt1170_evk/mimxrt1176/cm7

samples/subsys/video/capture/src/main.c

@@ -7,6 +7,7 @@
 #include <zephyr/kernel.h>
 #include <zephyr/device.h>
 
+#include <zephyr/drivers/display.h>
 #include <zephyr/drivers/video.h>
 
 #define LOG_LEVEL CONFIG_LOG_DEFAULT_LEVEL
@@ -15,39 +16,93 @@ LOG_MODULE_REGISTER(main);
 
 #define VIDEO_DEV_SW "VIDEO_SW_GENERATOR"
 
+#if DT_HAS_CHOSEN(zephyr_display)
+static inline int display_setup(const struct device *const display_dev, const uint32_t pixfmt)
+{
+	struct display_capabilities capabilities;
+	int ret = 0;
+
+	if (!device_is_ready(display_dev)) {
+		LOG_ERR("Device %s not found", display_dev->name);
+		return -ENODEV;
+	}
+
+	printk("\nDisplay device: %s\n", display_dev->name);
+
+	display_get_capabilities(display_dev, &capabilities);
+
+	printk("- Capabilities:\n");
+	printk("  x_resolution = %u, y_resolution = %u, supported_pixel_formats = %u\n"
+	       "  current_pixel_format = %u, current_orientation = %u\n\n",
+	       capabilities.x_resolution, capabilities.y_resolution,
+	       capabilities.supported_pixel_formats, capabilities.current_pixel_format,
+	       capabilities.current_orientation);
+
+	/* Set display pixel format to match the one in use by the camera */
+	switch (pixfmt) {
+	case VIDEO_PIX_FMT_RGB565:
+		ret = display_set_pixel_format(display_dev, PIXEL_FORMAT_BGR_565);
+		break;
+	case VIDEO_PIX_FMT_XRGB32:
+		ret = display_set_pixel_format(display_dev, PIXEL_FORMAT_ARGB_8888);
+		break;
+	default:
+		return -ENOTSUP;
+	}
+
+	if (ret) {
+		LOG_ERR("Unable to set display format");
+		return ret;
+	}
+
+	return display_blanking_off(display_dev);
+}
+
+static inline void video_display_frame(const struct device *const display_dev,
+				       const struct video_buffer *const vbuf,
+				       const struct video_format fmt)
+{
+	struct display_buffer_descriptor buf_desc;
+
+	buf_desc.buf_size = vbuf->bytesused;
+	buf_desc.width = fmt.width;
+	buf_desc.pitch = buf_desc.width;
+	buf_desc.height = fmt.height;
+
+	display_write(display_dev, 0, 0, &buf_desc, vbuf->buffer);
+}
+#endif
+
 int main(void)
 {
 	struct video_buffer *buffers[2], *vbuf;
 	struct video_format fmt;
 	struct video_caps caps;
-	const struct device *video;
 	unsigned int frame = 0;
 	size_t bsize;
 	int i = 0;
+	int err;
+
+#if DT_HAS_CHOSEN(zephyr_camera)
+	const struct device *const video_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_camera));
 
-	/* Default to software video pattern generator */
-	video = device_get_binding(VIDEO_DEV_SW);
-	if (video == NULL) {
-		LOG_ERR("Video device %s not found", VIDEO_DEV_SW);
+	if (!device_is_ready(video_dev)) {
+		LOG_ERR("%s: video device is not ready", video_dev->name);
 		return 0;
 	}
+#else
+	const struct device *const video_dev = device_get_binding(VIDEO_DEV_SW);
 
-	/* But would be better to use a real video device if any */
-#if defined(CONFIG_VIDEO_MCUX_CSI)
-	const struct device *const dev = DEVICE_DT_GET_ONE(nxp_imx_csi);
-
-	if (!device_is_ready(dev)) {
-		LOG_ERR("%s: device not ready.\n", dev->name);
+	if (video_dev == NULL) {
+		LOG_ERR("%s: video device not found or failed to initialized", VIDEO_DEV_SW);
 		return 0;
 	}
-
-	video = dev;
 #endif
 
-	printk("- Device name: %s\n", video->name);
+	printk("Video device: %s\n", video_dev->name);
 
 	/* Get capabilities */
-	if (video_get_caps(video, VIDEO_EP_OUT, &caps)) {
+	if (video_get_caps(video_dev, VIDEO_EP_OUT, &caps)) {
 		LOG_ERR("Unable to retrieve video capabilities");
 		return 0;
 	}
@@ -57,43 +112,58 @@ int main(void)
 		const struct video_format_cap *fcap = &caps.format_caps[i];
 		/* fourcc to string */
 		printk("  %c%c%c%c width [%u; %u; %u] height [%u; %u; %u]\n",
-		       (char)fcap->pixelformat,
-		       (char)(fcap->pixelformat >> 8),
-		       (char)(fcap->pixelformat >> 16),
-		       (char)(fcap->pixelformat >> 24),
-		       fcap->width_min, fcap->width_max, fcap->width_step,
-		       fcap->height_min, fcap->height_max, fcap->height_step);
+		       (char)fcap->pixelformat, (char)(fcap->pixelformat >> 8),
+		       (char)(fcap->pixelformat >> 16), (char)(fcap->pixelformat >> 24),
+		       fcap->width_min, fcap->width_max, fcap->width_step, fcap->height_min,
+		       fcap->height_max, fcap->height_step);
 		i++;
 	}
 
 	/* Get default/native format */
-	if (video_get_format(video, VIDEO_EP_OUT, &fmt)) {
+	if (video_get_format(video_dev, VIDEO_EP_OUT, &fmt)) {
 		LOG_ERR("Unable to retrieve video format");
 		return 0;
 	}
 
 	printk("- Default format: %c%c%c%c %ux%u\n", (char)fmt.pixelformat,
-	       (char)(fmt.pixelformat >> 8),
-	       (char)(fmt.pixelformat >> 16),
-	       (char)(fmt.pixelformat >> 24),
-	       fmt.width, fmt.height);
+	       (char)(fmt.pixelformat >> 8), (char)(fmt.pixelformat >> 16),
+	       (char)(fmt.pixelformat >> 24), fmt.width, fmt.height);
+
+#if DT_HAS_CHOSEN(zephyr_display)
+	const struct device *const display_dev = DEVICE_DT_GET(DT_CHOSEN(zephyr_display));
+
+	if (!device_is_ready(display_dev)) {
+		LOG_ERR("%s: display device not ready.", display_dev->name);
+		return 0;
+	}
+
+	err = display_setup(display_dev, fmt.pixelformat);
+	if (err) {
+		LOG_ERR("Unable to set up display");
+		return err;
+	}
+#endif
 
 	/* Size to allocate for each buffer */
 	bsize = fmt.pitch * fmt.height;
 
 	/* Alloc video buffers and enqueue for capture */
 	for (i = 0; i < ARRAY_SIZE(buffers); i++) {
-		buffers[i] = video_buffer_alloc(bsize);
+		/*
+		 * For some hardwares, such as the PxP used on i.MX RT1170 to do image rotation,
+		 * buffer alignment is needed in order to achieve the best performance
+		 */
+		buffers[i] = video_buffer_aligned_alloc(bsize, CONFIG_VIDEO_BUFFER_POOL_ALIGN);
 		if (buffers[i] == NULL) {
 			LOG_ERR("Unable to alloc video buffer");
 			return 0;
 		}
 
-		video_enqueue(video, VIDEO_EP_OUT, buffers[i]);
+		video_enqueue(video_dev, VIDEO_EP_OUT, buffers[i]);
 	}
 
 	/* Start video capture */
-	if (video_stream_start(video)) {
+	if (video_stream_start(video_dev)) {
 		LOG_ERR("Unable to start capture (interface)");
 		return 0;
 	}
@@ -102,18 +172,20 @@ int main(void)
 
 	/* Grab video frames */
 	while (1) {
-		int err;
-
-		err = video_dequeue(video, VIDEO_EP_OUT, &vbuf, K_FOREVER);
+		err = video_dequeue(video_dev, VIDEO_EP_OUT, &vbuf, K_FOREVER);
 		if (err) {
 			LOG_ERR("Unable to dequeue video buf");
 			return 0;
 		}
 
-		printk("\rGot frame %u! size: %u; timestamp %u ms",
-		       frame++, vbuf->bytesused, vbuf->timestamp);
+		printk("Got frame %u! size: %u; timestamp %u ms\n", frame++, vbuf->bytesused,
+		       vbuf->timestamp);
+
+#if DT_HAS_CHOSEN(zephyr_display)
+		video_display_frame(display_dev, vbuf, fmt);
+#endif
 
-		err = video_enqueue(video, VIDEO_EP_OUT, vbuf);
+		err = video_enqueue(video_dev, VIDEO_EP_OUT, vbuf);
 		if (err) {
 			LOG_ERR("Unable to requeue video buf");
 			return 0;