Skip to content

Modify NXP's CSI driver to support both rt11xx and rt10xx #72420

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
May 23, 2024
Merged

Modify NXP's CSI driver to support both rt11xx and rt10xx #72420

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
de2be86
include: video: Add XYUV32 pixel format
ngphibang May 11, 2024
668b5c6
drivers: video: csi: Add support for i.MX RT11XX
ngphibang May 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
77 changes: 75 additions & 2 deletions drivers/video/video_mcux_csi.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,9 @@ static inline unsigned int video_pix_fmt_bpp(uint32_t pixelformat)
case VIDEO_PIX_FMT_RGB565:
case VIDEO_PIX_FMT_YUYV:
return 2;
case VIDEO_PIX_FMT_XRGB32:
case VIDEO_PIX_FMT_XYUV32:
return 4;
default:
return 0;
}
Expand Down Expand Up @@ -113,23 +116,62 @@ static void __frame_done_cb(CSI_Type *base, csi_handle_t *handle, status_t statu
return;
}

#if defined(CONFIG_VIDEO_MCUX_MIPI_CSI2RX)
K_HEAP_DEFINE(csi_heap, 1000);
static struct video_format_cap *fmts;
/*
* On i.MX RT11xx SoCs which have MIPI CSI-2 Rx, image data from the camera sensor after passing
* through the pipeline (MIPI CSI-2 Rx --> Video Mux --> CSI) will be implicitly converted to a
* 32-bits pixel format. For example, an input in RGB565 or YUYV (2-bytes format) will become a
* XRGB32 or XYUV32 (4-bytes format) respectively, at the output of the CSI.
*/
static inline void video_pix_fmt_convert(struct video_format *fmt, bool isGetFmt)
{
switch (fmt->pixelformat) {
case VIDEO_PIX_FMT_XRGB32:
fmt->pixelformat = isGetFmt ? VIDEO_PIX_FMT_XRGB32 : VIDEO_PIX_FMT_RGB565;
break;
case VIDEO_PIX_FMT_XYUV32:
fmt->pixelformat = isGetFmt ? VIDEO_PIX_FMT_XYUV32 : VIDEO_PIX_FMT_YUYV;
break;
case VIDEO_PIX_FMT_RGB565:
fmt->pixelformat = isGetFmt ? VIDEO_PIX_FMT_XRGB32 : VIDEO_PIX_FMT_RGB565;
break;
case VIDEO_PIX_FMT_YUYV:
fmt->pixelformat = isGetFmt ? VIDEO_PIX_FMT_XYUV32 : VIDEO_PIX_FMT_YUYV;
break;
}

fmt->pitch = fmt->width * video_pix_fmt_bpp(fmt->pixelformat);
}
#endif

static int video_mcux_csi_set_fmt(const struct device *dev, enum video_endpoint_id ep,
struct video_format *fmt)
{
const struct video_mcux_csi_config *config = dev->config;
struct video_mcux_csi_data *data = dev->data;
unsigned int bpp = video_pix_fmt_bpp(fmt->pixelformat);
status_t ret;
struct video_format format = *fmt;

if (!bpp || ep != VIDEO_EP_OUT) {
return -EINVAL;
}

data->csi_config.bytesPerPixel = bpp;
data->csi_config.linePitch_Bytes = fmt->pitch;
#if defined(CONFIG_VIDEO_MCUX_MIPI_CSI2RX)
if (fmt->pixelformat != VIDEO_PIX_FMT_XRGB32 && fmt->pixelformat != VIDEO_PIX_FMT_XYUV32) {
return -ENOTSUP;
}
video_pix_fmt_convert(&format, false);
data->csi_config.dataBus = kCSI_DataBus24Bit;
Copy link
Contributor

@danieldegrasse danieldegrasse May 7, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe instead of making this dependent on Kconfig we should move the dataBus configuration to devicetree? Not sure if all cameras are 8bit bus width, or if some would use more pins

Copy link
Contributor Author

@ngphibang ngphibang May 11, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We can't make this configurable in devicetree. The databus is either 8 bits or 24 bits depending on the path of image data. If it passes by mipi csi2rx, data are implicitly converted to 32-bit format and 24 bits need to be set for it to work.

#else
data->csi_config.dataBus = kCSI_DataBus8Bit;
#endif
data->csi_config.polarityFlags = kCSI_HsyncActiveHigh | kCSI_DataLatchOnRisingEdge;
data->csi_config.workMode = kCSI_GatedClockMode; /* use VSYNC, HSYNC, and PIXCLK */
data->csi_config.dataBus = kCSI_DataBus8Bit;
data->csi_config.useExtVsync = true;
data->csi_config.height = fmt->height;
data->csi_config.width = fmt->width;
Expand All @@ -144,7 +186,7 @@ static int video_mcux_csi_set_fmt(const struct device *dev, enum video_endpoint_
return -EIO;
}

if (config->source_dev && video_set_format(config->source_dev, ep, fmt)) {
if (config->source_dev && video_set_format(config->source_dev, ep, &format)) {
return -EIO;
}

Expand All @@ -161,6 +203,9 @@ static int video_mcux_csi_get_fmt(const struct device *dev, enum video_endpoint_
}

if (config->source_dev && !video_get_format(config->source_dev, ep, fmt)) {
#if defined(CONFIG_VIDEO_MCUX_MIPI_CSI2RX)
video_pix_fmt_convert(fmt, true);
#endif
/* align CSI with source fmt */
return video_mcux_csi_set_fmt(dev, ep, fmt);
}
Expand Down Expand Up @@ -317,6 +362,34 @@ static int video_mcux_csi_get_caps(const struct device *dev, enum video_endpoint
/* Just forward to source dev for now */
if (config->source_dev) {
err = video_get_caps(config->source_dev, ep, caps);
#if defined(CONFIG_VIDEO_MCUX_MIPI_CSI2RX)
/*
* On i.MX RT11xx SoCs which have MIPI CSI-2 Rx, image data from the camera sensor
* after passing through the pipeline (MIPI CSI-2 Rx --> Video Mux --> CSI) will be
* implicitly converted to a 32-bits pixel format. For example, an input in RGB565
* or YUYV (2-bytes format) will become an XRGB32 or XYUV32 (4-bytes format)
* respectively, at the output of the CSI. So, we change the pixel formats of the
* source caps to reflect this.
*/
int ind = 0;

while (caps->format_caps[ind].pixelformat) {
ind++;
}
k_heap_free(&csi_heap, fmts);
fmts = k_heap_alloc(&csi_heap, (ind + 1) * sizeof(struct video_format_cap),
K_FOREVER);

for (int i = 0; i <= ind; i++) {
memcpy(&fmts[i], &caps->format_caps[i], sizeof(fmts[i]));
if (fmts[i].pixelformat == VIDEO_PIX_FMT_RGB565) {
fmts[i].pixelformat = VIDEO_PIX_FMT_XRGB32;
} else if (fmts[i].pixelformat == VIDEO_PIX_FMT_YUYV) {
fmts[i].pixelformat = VIDEO_PIX_FMT_XYUV32;
}
}
caps->format_caps = fmts;
#endif
}

/* NXP MCUX CSI request at least 2 buffer before starting */
Expand Down
3 changes: 3 additions & 0 deletions include/zephyr/drivers/video.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -638,6 +638,9 @@ void video_buffer_release(struct video_buffer *buf);
/** YUYV pixel format */
#define VIDEO_PIX_FMT_YUYV video_fourcc('Y', 'U', 'Y', 'V') /* 16 Y0-Cb0 Y1-Cr0 */

/** XYUV32 pixel format */
#define VIDEO_PIX_FMT_XYUV32 video_fourcc('X', 'Y', 'U', 'V') /* 32 XYUV-8-8-8-8 */

/**
*
* @}
Expand Down