From 690ceb1b09979873a75f7591db6d490ed0dce4a8 Mon Sep 17 00:00:00 2001 From: Mohammad Massoudi Date: Wed, 3 Sep 2025 16:07:26 +0330 Subject: [PATCH 1/7] include: driver: video: add h264 pixel format support Add H264 pixel format support. Signed-off-by: Mohammad Massoudi --- include/zephyr/drivers/video.h | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index d4407420d0859..4e476e4be8f5d 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -1768,6 +1768,16 @@ int64_t video_get_csi_link_freq(const struct device *dev, uint8_t bpp, uint8_t l */ #define VIDEO_PIX_FMT_JPEG VIDEO_FOURCC('J', 'P', 'E', 'G') +/** + * H264 with start code + */ +#define VIDEO_PIX_FMT_H264 VIDEO_FOURCC('H', '2', '6', '4') + +/** + * H264 without start code + */ +#define VIDEO_PIX_FMT_H264_NO_SC VIDEO_FOURCC('A', 'V', 'C', '1') + /** * @} */ From 77d8c221ed28bbabec2ef4bcaab4048632952cec Mon Sep 17 00:00:00 2001 From: Hugues Fruchet Date: Wed, 1 Oct 2025 10:05:49 +0200 Subject: [PATCH 2/7] include: video: add size field to video_format structure Add size field to the video_format structure which needs to be set by the driver and exposed to the application. For uncompressed formats, this is the size of the raw data buffer in bytes, which could be the whole raw image or a portion of the raw image in cases the receiver / dma supports it. For compressed formats, this is the estimated maximum number of bytes required to hold a complete compressed frame. Signed-off-by: Hugues Fruchet Signed-off-by: Phi Bang Nguyen --- include/zephyr/drivers/video.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/include/zephyr/drivers/video.h b/include/zephyr/drivers/video.h index 4e476e4be8f5d..87eb0314a48d8 100644 --- a/include/zephyr/drivers/video.h +++ b/include/zephyr/drivers/video.h @@ -79,6 +79,17 @@ struct video_format { * the next row (>=width). */ uint32_t pitch; + /** + * @brief size of the buffer in bytes, need to be set by the drivers + * + * For uncompressed formats, this is the size of the raw data buffer in bytes, + * which could be the whole raw image or a portion of the raw image in cases + * the receiver / dma supports it. + * + * For compressed formats, this is the maximum number of bytes required to + * hold a complete compressed frame, estimated for the worst case. + */ + uint32_t size; }; /** From 72add74dfeeb0c03ea6e5a5807766915e156b06b Mon Sep 17 00:00:00 2001 From: Hugues Fruchet Date: Wed, 8 Oct 2025 20:57:14 +0200 Subject: [PATCH 3/7] doc: release-note-4.3: document size field added to video_format struct Document introduction of the new size field inside video_format struct as visible on the header. Signed-off-by: Hugues Fruchet --- doc/releases/release-notes-4.3.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/doc/releases/release-notes-4.3.rst b/doc/releases/release-notes-4.3.rst index 906002ffb1bd1..c4d94c0f75bae 100644 --- a/doc/releases/release-notes-4.3.rst +++ b/doc/releases/release-notes-4.3.rst @@ -276,6 +276,10 @@ New APIs and options * :c:macro:`__deprecated_version` +* Video + + * :c:member:`video_format.size` field + .. zephyr-keep-sorted-stop New Boards From bf3e2a57520732f56a322702a44ac91caf582861 Mon Sep 17 00:00:00 2001 From: Hugues Fruchet Date: Wed, 18 Jun 2025 11:46:55 +0200 Subject: [PATCH 4/7] dts-bindings: video: addition of stm32 venc description Addition of description for the STM32 Video encoder (VENC). Signed-off-by: Hugues Fruchet --- dts/bindings/video/st,stm32-venc.yaml | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 dts/bindings/video/st,stm32-venc.yaml diff --git a/dts/bindings/video/st,stm32-venc.yaml b/dts/bindings/video/st,stm32-venc.yaml new file mode 100644 index 0000000000000..c90d79fecffec --- /dev/null +++ b/dts/bindings/video/st,stm32-venc.yaml @@ -0,0 +1,23 @@ +# +# Copyright (c) 2025 STMicroelectronics. +# +# SPDX-License-Identifier: Apache-2.0 +# +title: STM32 video encoder (VENC) + +description: | + STMicroelectronics STM32 video encoder peripheral (VENC). + +compatible: "st,stm32-venc" + +include: [base.yaml, reset-device.yaml] + +properties: + interrupts: + required: true + + clocks: + required: true + + resets: + required: true From cd68aef1dedfa3b8562774cb157ea07b665441d4 Mon Sep 17 00:00:00 2001 From: Hugues Fruchet Date: Wed, 18 Jun 2025 11:27:42 +0200 Subject: [PATCH 5/7] drivers: video: introduction of the stm32 venc driver The STM32 video encoder (VENC) peripheral is a hardware accelerator allowing to compress RGB/YUV frames into H264 video bitstream chunks. Signed-off-by: Hugues Fruchet --- drivers/video/CMakeLists.txt | 1 + drivers/video/Kconfig | 2 + drivers/video/Kconfig.stm32_venc | 24 + drivers/video/video_stm32_venc.c | 914 ++++++++++++++++++++ tests/drivers/build_all/video/testcase.yaml | 3 + 5 files changed, 944 insertions(+) create mode 100644 drivers/video/Kconfig.stm32_venc create mode 100644 drivers/video/video_stm32_venc.c diff --git a/drivers/video/CMakeLists.txt b/drivers/video/CMakeLists.txt index 7a03156165ce0..2f6a3244b5ad9 100644 --- a/drivers/video/CMakeLists.txt +++ b/drivers/video/CMakeLists.txt @@ -15,6 +15,7 @@ zephyr_library_sources_ifdef(CONFIG_VIDEO_OV7725 ov7725.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_OV2640 ov2640.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_GC2145 gc2145.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_STM32_DCMI video_stm32_dcmi.c) +zephyr_library_sources_ifdef(CONFIG_VIDEO_STM32_VENC video_stm32_venc.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_OV5640 ov5640.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_OV7670 ov7670.c) zephyr_library_sources_ifdef(CONFIG_VIDEO_OV9655 ov9655.c) diff --git a/drivers/video/Kconfig b/drivers/video/Kconfig index 7b1f57a83c487..9d7fa8417b35d 100644 --- a/drivers/video/Kconfig +++ b/drivers/video/Kconfig @@ -76,6 +76,8 @@ source "drivers/video/Kconfig.ov2640" source "drivers/video/Kconfig.stm32_dcmi" +source "drivers/video/Kconfig.stm32_venc" + source "drivers/video/Kconfig.ov5640" source "drivers/video/Kconfig.ov7670" diff --git a/drivers/video/Kconfig.stm32_venc b/drivers/video/Kconfig.stm32_venc new file mode 100644 index 0000000000000..c0e19575021a8 --- /dev/null +++ b/drivers/video/Kconfig.stm32_venc @@ -0,0 +1,24 @@ +# STM32 VENC driver configuration options + +# Copyright (c) 2025 STMicroelectronics. +# SPDX-License-Identifier: Apache-2.0 + +config VIDEO_STM32_VENC + bool "STM32 video encoder (VENC) driver" + default y + depends on DT_HAS_ST_STM32_VENC_ENABLED + select HAS_STM32LIB + select USE_STM32_LL_VENC + select USE_STM32_HAL_RIF if SOC_SERIES_STM32N6X + select RESET + help + Enable driver for STM32 video encoder peripheral. + +if VIDEO_STM32_VENC + +# See hal_stm32/lib/vc8000nanoe/inc/encdebug.h +module = VC8000NANOE +module-str = vc8000nanoe +source "subsys/logging/Kconfig.template.log_config" + +endif diff --git a/drivers/video/video_stm32_venc.c b/drivers/video/video_stm32_venc.c new file mode 100644 index 0000000000000..4f26977dcce10 --- /dev/null +++ b/drivers/video/video_stm32_venc.c @@ -0,0 +1,914 @@ +/* + * Copyright (c) 2025 STMicroelectronics. + * + * SPDX-License-Identifier: Apache-2.0 + */ + +#define DT_DRV_COMPAT st_stm32_venc + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +LOG_MODULE_REGISTER(stm32_venc, CONFIG_VIDEO_LOG_LEVEL); + +#define VENC_DEFAULT_WIDTH 320 +#define VENC_DEFAULT_HEIGHT 240 +#define VENC_DEFAULT_IN_FMT VIDEO_PIX_FMT_NV12 +#define VENC_DEFAULT_OUT_FMT VIDEO_PIX_FMT_H264 +#define VENC_DEFAULT_FRAMERATE 30 +#define VENC_DEFAULT_LEVEL H264ENC_LEVEL_4 +#define VENC_DEFAULT_QP 25 +#define VENC_ESTIMATED_COMPRESSION_RATIO 10 + +#define ALIGNMENT_INCR 8UL + +#define EWL_HEAP_ALIGNED_ALLOC(size)\ + shared_multi_heap_aligned_alloc(CONFIG_VIDEO_BUFFER_SMH_ATTRIBUTE, ALIGNMENT_INCR, (size)) +#define EWL_HEAP_ALIGNED_FREE(block) shared_multi_heap_free(block) + +#define EWL_TIMEOUT 100UL + +#define MEM_CHUNKS 32 + +#define NUM_SLICES_READY_MASK GENMASK(23, 16) +#define LOW_LATENCY_HW_ITF_EN 29 + +typedef void (*irq_config_func_t)(const struct device *dev); + +struct stm32_venc_config { + mm_reg_t reg; + const struct stm32_pclken pclken; + const struct reset_dt_spec reset; + irq_config_func_t irq_config; +}; + +struct stm32_venc_ewl { + uint32_t client_type; + uint32_t *chunks[MEM_CHUNKS]; + uint32_t *aligned_chunks[MEM_CHUNKS]; + uint32_t total_chunks; + const struct stm32_venc_config *config; + struct k_sem complete; + uint32_t irq_status; + uint32_t irq_cnt; + uint32_t mem_cnt; +}; + +static struct stm32_venc_ewl ewl_instance; + +struct stm32_venc_data { + const struct device *dev; + struct k_mutex lock; + struct video_format in_fmt; + struct video_format out_fmt; + struct k_fifo in_fifo_in; + struct k_fifo in_fifo_out; + struct k_fifo out_fifo_in; + struct k_fifo out_fifo_out; + struct video_buffer *vbuf; + H264EncInst encoder; + uint32_t frame_nb; + bool resync; +}; + +static H264EncPictureType to_h264pixfmt(uint32_t pixelformat) +{ + switch (pixelformat) { + case VIDEO_PIX_FMT_NV12: + return H264ENC_YUV420_SEMIPLANAR; + case VIDEO_PIX_FMT_RGB565: + return H264ENC_RGB565; + default: + __ASSERT_NO_MSG(false); + return H264ENC_YUV420_SEMIPLANAR; + } +} + +u32 EWLReadAsicID(void) +{ + const struct stm32_venc_config *config = ewl_instance.config; + + return sys_read32(config->reg + BASE_HEncASIC); +} + +EWLHwConfig_t EWLReadAsicConfig(void) +{ + const struct stm32_venc_config *config = ewl_instance.config; + EWLHwConfig_t cfg_info; + uint32_t cfgval, cfgval2; + + cfgval = sys_read32(config->reg + BASE_HEncSynth); + cfgval2 = sys_read32(config->reg + BASE_HEncSynth1); + + cfg_info = EWLBuildHwConfig(cfgval, cfgval2); + + LOG_DBG("maxEncodedWidth = %d", cfg_info.maxEncodedWidth); + LOG_DBG("h264Enabled = %d", cfg_info.h264Enabled); + LOG_DBG("jpegEnabled = %d", cfg_info.jpegEnabled); + LOG_DBG("vp8Enabled = %d", cfg_info.vp8Enabled); + LOG_DBG("vsEnabled = %d", cfg_info.vsEnabled); + LOG_DBG("rgbEnabled = %d", cfg_info.rgbEnabled); + LOG_DBG("searchAreaSmall = %d", cfg_info.searchAreaSmall); + LOG_DBG("scalingEnabled = %d", cfg_info.scalingEnabled); + LOG_DBG("address64bits = %d", cfg_info.addr64Support); + LOG_DBG("denoiseEnabled = %d", cfg_info.dnfSupport); + LOG_DBG("rfcEnabled = %d", cfg_info.rfcSupport); + LOG_DBG("instanctEnabled = %d", cfg_info.instantSupport); + LOG_DBG("busType = %d", cfg_info.busType); + LOG_DBG("synthesisLanguage = %d", cfg_info.synthesisLanguage); + LOG_DBG("busWidth = %d", cfg_info.busWidth * 32); + + return cfg_info; +} + +const void *EWLInit(EWLInitParam_t *param) +{ + __ASSERT_NO_MSG(param != NULL); + __ASSERT_NO_MSG(param->clientType == EWL_CLIENT_TYPE_H264_ENC); + + /* sync */ + k_sem_init(&ewl_instance.complete, 0, 1); + + /* set client type */ + ewl_instance.client_type = param->clientType; + ewl_instance.irq_cnt = 0; + + return (void *)&ewl_instance; +} + +i32 EWLRelease(const void *inst) +{ + ARG_UNUSED(inst); + + return EWL_OK; +} + +void EWLWriteReg(const void *instance, uint32_t offset, uint32_t val) +{ + struct stm32_venc_ewl *inst = (struct stm32_venc_ewl *)instance; + const struct stm32_venc_config *config = inst->config; + + sys_write32(val, config->reg + offset); +} + +void EWLEnableHW(const void *instance, uint32_t offset, uint32_t val) +{ + struct stm32_venc_ewl *inst = (struct stm32_venc_ewl *)instance; + const struct stm32_venc_config *config = inst->config; + + sys_write32(val, config->reg + offset); +} + +void EWLDisableHW(const void *instance, uint32_t offset, uint32_t val) +{ + struct stm32_venc_ewl *inst = (struct stm32_venc_ewl *)instance; + const struct stm32_venc_config *config = inst->config; + + sys_write32(val, config->reg + offset); +} + +uint32_t EWLReadReg(const void *instance, uint32_t offset) +{ + struct stm32_venc_ewl *inst = (struct stm32_venc_ewl *)instance; + const struct stm32_venc_config *config = inst->config; + + return sys_read32(config->reg + offset); +} + +i32 EWLMallocRefFrm(const void *instance, uint32_t size, EWLLinearMem_t *info) +{ + return EWLMallocLinear(instance, size, info); +} + +void EWLFreeRefFrm(const void *instance, EWLLinearMem_t *info) +{ + EWLFreeLinear(instance, info); +} + +i32 EWLMallocLinear(const void *instance, uint32_t size, EWLLinearMem_t *info) +{ + struct stm32_venc_ewl *inst = (struct stm32_venc_ewl *)instance; + + __ASSERT_NO_MSG(inst != NULL); + __ASSERT_NO_MSG(info != NULL); + + /* align size */ + uint32_t size_aligned = ROUND_UP(size, ALIGNMENT_INCR); + + info->size = size_aligned; + + /* allocate */ + inst->chunks[inst->total_chunks] = (uint32_t *)EWL_HEAP_ALIGNED_ALLOC(size_aligned); + if (inst->chunks[inst->total_chunks] == NULL) { + LOG_DBG("unable to allocate %8d bytes", size_aligned); + return EWL_ERROR; + } + + /* align given allocated buffer */ + inst->aligned_chunks[inst->total_chunks] = + (uint32_t *)ROUND_UP((uint32_t)inst->chunks[inst->total_chunks], ALIGNMENT_INCR); + /* put the aligned pointer in the return structure */ + info->virtualAddress = inst->aligned_chunks[inst->total_chunks]; + if (info->virtualAddress == NULL) { + LOG_DBG("unable to get chunk for %8d bytes", size_aligned); + EWL_HEAP_ALIGNED_FREE(inst->chunks[inst->total_chunks]); + return EWL_ERROR; + } + inst->total_chunks++; + + /* bus address is the same as virtual address because no MMU */ + info->busAddress = (ptr_t)info->virtualAddress; + + inst->mem_cnt += size; + LOG_DBG("allocated %8d bytes --> %p / 0x%x. Total : %d", size_aligned, + (void *)info->virtualAddress, info->busAddress, inst->mem_cnt); + + return EWL_OK; +} + +void EWLFreeLinear(const void *instance, EWLLinearMem_t *info) +{ + struct stm32_venc_ewl *inst = (struct stm32_venc_ewl *)instance; + + __ASSERT_NO_MSG(inst != NULL); + __ASSERT_NO_MSG(info != NULL); + + /* find the pointer corresponding to the aligned buffer */ + for (uint32_t i = 0; i < inst->total_chunks; i++) { + if (inst->aligned_chunks[i] == info->virtualAddress) { + if (inst->chunks[i] != NULL) { + EWL_HEAP_ALIGNED_FREE(inst->chunks[i]); + } + break; + } + } + info->virtualAddress = NULL; + info->busAddress = 0; + info->size = 0; +} + +i32 EWLReserveHw(const void *inst) +{ + __ASSERT_NO_MSG(inst != NULL); + + return EWL_OK; +} + +void EWLReleaseHw(const void *inst) +{ + __ASSERT_NO_MSG(inst != NULL); +} + +void *EWLmalloc(uint32_t n) +{ + struct stm32_venc_ewl *inst = &ewl_instance; + void *p = NULL; + + p = EWL_HEAP_ALIGNED_ALLOC(n); + if (p == NULL) { + LOG_ERR("alloc failed for size=%d", n); + return NULL; + } + + inst->mem_cnt += n; + LOG_DBG("%8d bytes --> %p, total : %d", n, p, inst->mem_cnt); + + return p; +} + +void EWLfree(void *p) +{ + if (p != NULL) { + EWL_HEAP_ALIGNED_FREE(p); + } +} + +void *EWLcalloc(uint32_t n, uint32_t s) +{ + void *p = EWLmalloc(n * s); + + if (p != NULL) { + EWLmemset(p, 0, n * s); + } + + return p; +} + +void *EWLmemcpy(void *d, const void *s, uint32_t n) +{ + return memcpy(d, s, (size_t)n); +} + +void *EWLmemset(void *d, i32 c, uint32_t n) +{ + return memset(d, c, (size_t)n); +} + +int EWLmemcmp(const void *s1, const void *s2, uint32_t n) +{ + return memcmp(s1, s2, n); +} + +i32 EWLWaitHwRdy(const void *instance, uint32_t *slicesReady) +{ + struct stm32_venc_ewl *inst = (struct stm32_venc_ewl *)instance; + const struct stm32_venc_config *config = inst->config; + uint32_t ret = EWL_HW_WAIT_TIMEOUT; + volatile uint32_t irq_stats; + uint32_t prevSlicesReady = 0; + k_timepoint_t timeout = sys_timepoint_calc(K_MSEC(EWL_TIMEOUT)); + uint32_t start = sys_clock_tick_get_32(); + + __ASSERT_NO_MSG(inst != NULL); + + /* check how to clear IRQ flags for VENC */ + uint32_t clrByWrite1 = EWLReadReg(inst, BASE_HWFuse2) & HWCFGIrqClearSupport; + + do { + irq_stats = sys_read32(config->reg + BASE_HEncIRQ); + /* get the number of completed slices from ASIC registers. */ + if (slicesReady != NULL && *slicesReady > prevSlicesReady) { + *slicesReady = FIELD_GET(NUM_SLICES_READY_MASK, + sys_read32(config->reg + BASE_HEncControl7)); + } + + LOG_DBG("IRQ stat = %08x", irq_stats); + + uint32_t hw_handshake_status = IS_BIT_SET( + sys_read32(config->reg + BASE_HEncInstantInput), LOW_LATENCY_HW_ITF_EN); + + /* ignore the irq status of input line buffer in hw handshake mode */ + if ((irq_stats == ASIC_STATUS_LINE_BUFFER_DONE) && (hw_handshake_status != 0UL)) { + sys_write32(ASIC_STATUS_FUSE, config->reg + BASE_HEncIRQ); + continue; + } + + if ((irq_stats & ASIC_STATUS_ALL) != 0UL) { + /* clear IRQ and slice ready status */ + uint32_t clr_stats; + + irq_stats &= ~(ASIC_STATUS_SLICE_READY | ASIC_IRQ_LINE); + + if (clrByWrite1 != 0UL) { + clr_stats = ASIC_STATUS_SLICE_READY | ASIC_IRQ_LINE; + } else { + clr_stats = irq_stats; + } + + sys_write32(clr_stats, config->reg + BASE_HEncIRQ); + ret = EWL_OK; + break; + } + + if (slicesReady != NULL && *slicesReady > prevSlicesReady) { + ret = EWL_OK; + break; + } + + } while (!sys_timepoint_expired(timeout)); + + LOG_DBG("encoding = %d ms", k_ticks_to_ms_ceil32(sys_clock_tick_get_32() - start)); + + if (slicesReady != NULL) { + LOG_DBG("slicesReady = %d", *slicesReady); + } + + return ret; +} + +void EWLassert(bool expr, const char *str_expr, const char *file, unsigned int line) +{ + __ASSERT(expr, "ASSERTION FAIL [%s] @ %s:%d", str_expr, file, line); +} + +/* Set CONFIG_VC8000NANOE_LOG_LEVEL_DBG to enable library tracing */ +void EWLtrace(const char *s) +{ + printk("%s\n", s); +} + +void EWLtraceparam(const char *fmt, const char *param, unsigned int val) +{ + printk(fmt, param, val); +} + +static int stm32_venc_enable_clock(const struct device *dev) +{ + const struct stm32_venc_config *config = dev->config; + const struct device *clk = DEVICE_DT_GET(STM32_CLOCK_CONTROL_NODE); + + if (!device_is_ready(clk)) { + LOG_ERR("clock control device not ready"); + return -ENODEV; + } + + if (clock_control_on(clk, (clock_control_subsys_t)&config->pclken) != 0) { + return -EIO; + } + + return 0; +} + +static int stm32_venc_set_fmt(const struct device *dev, struct video_format *fmt) +{ + struct stm32_venc_data *data = dev->data; + + if (fmt->type == VIDEO_BUF_TYPE_INPUT) { + if ((fmt->pixelformat != VIDEO_PIX_FMT_NV12) && + (fmt->pixelformat != VIDEO_PIX_FMT_RGB565)) { + LOG_ERR("invalid input pixel format"); + return -EINVAL; + } + + fmt->pitch = fmt->width * video_bits_per_pixel(fmt->pixelformat) / BITS_PER_BYTE; + data->in_fmt = *fmt; + } else { + if (fmt->pixelformat != VIDEO_PIX_FMT_H264) { + LOG_ERR("invalid output pixel format"); + return -EINVAL; + } + + fmt->size = ROUND_UP(fmt->width * fmt->height / VENC_ESTIMATED_COMPRESSION_RATIO, + ALIGNMENT_INCR); + + data->out_fmt = *fmt; + } + + return 0; +} + +static int stm32_venc_get_fmt(const struct device *dev, struct video_format *fmt) +{ + struct stm32_venc_data *data = dev->data; + + if (fmt->type == VIDEO_BUF_TYPE_INPUT) { + *fmt = data->in_fmt; + } else { + *fmt = data->out_fmt; + } + + return 0; +} + +static int encoder_prepare(struct stm32_venc_data *data) +{ + H264EncRet ret; + H264EncConfig cfg = {0}; + H264EncPreProcessingCfg preproc_cfg = {0}; + H264EncRateCtrl ratectrl_cfg = {0}; + H264EncCodingCtrl codingctrl_cfg = {0}; + + data->frame_nb = 0; + + /* set config to 1 reference frame */ + cfg.refFrameAmount = 1; + /* frame rate */ + cfg.frameRateDenom = 1; + cfg.frameRateNum = VENC_DEFAULT_FRAMERATE; + /* image resolution */ + cfg.width = data->out_fmt.width; + cfg.height = data->out_fmt.height; + /* stream type */ + cfg.streamType = H264ENC_BYTE_STREAM; + + /* encoding level*/ + cfg.level = VENC_DEFAULT_LEVEL; + cfg.svctLevel = 0; + cfg.viewMode = H264ENC_BASE_VIEW_SINGLE_BUFFER; + + ret = H264EncInit(&cfg, &data->encoder); + if (ret != H264ENC_OK) { + LOG_ERR("H264EncInit error=%d", ret); + return -EIO; + } + + /* set format conversion for preprocessing */ + ret = H264EncGetPreProcessing(data->encoder, &preproc_cfg); + if (ret != H264ENC_OK) { + LOG_ERR("H264EncGetPreProcessing error=%d", ret); + return -EIO; + } + preproc_cfg.inputType = to_h264pixfmt(data->in_fmt.pixelformat); + ret = H264EncSetPreProcessing(data->encoder, &preproc_cfg); + if (ret != H264ENC_OK) { + LOG_ERR("H264EncSetPreProcessing error=%d", ret); + return -EIO; + } + + /* setup coding ctrl */ + ret = H264EncGetCodingCtrl(data->encoder, &codingctrl_cfg); + if (ret != H264ENC_OK) { + LOG_ERR("H264EncGetCodingCtrl error=%d", ret); + return -EIO; + } + + ret = H264EncSetCodingCtrl(data->encoder, &codingctrl_cfg); + if (ret != H264ENC_OK) { + LOG_ERR("H264EncSetCodingCtrl error=%d", ret); + return -EIO; + } + + /* set bit rate configuration */ + ret = H264EncGetRateCtrl(data->encoder, &ratectrl_cfg); + if (ret != H264ENC_OK) { + LOG_ERR("H264EncGetRateCtrl error=%d", ret); + return -EIO; + } + + /* Constant bitrate */ + ratectrl_cfg.pictureRc = 0; + ratectrl_cfg.mbRc = 0; + ratectrl_cfg.pictureSkip = 0; + ratectrl_cfg.hrd = 0; + ratectrl_cfg.qpHdr = VENC_DEFAULT_QP; + ratectrl_cfg.qpMin = ratectrl_cfg.qpHdr; + ratectrl_cfg.qpMax = ratectrl_cfg.qpHdr; + + ret = H264EncSetRateCtrl(data->encoder, &ratectrl_cfg); + if (ret != H264ENC_OK) { + LOG_ERR("H264EncSetRateCtrl error=%d", ret); + return -EIO; + } + + return 0; +} + +static int encoder_start(struct stm32_venc_data *data, struct video_buffer *output) +{ + H264EncRet ret; + H264EncIn encIn = {0}; + H264EncOut encOut = {0}; + + encIn.pOutBuf = (uint32_t *)output->buffer; + encIn.busOutBuf = (uint32_t)encIn.pOutBuf; + encIn.outBufSize = output->size; + + /* create stream */ + ret = H264EncStrmStart(data->encoder, &encIn, &encOut); + if (ret != H264ENC_OK) { + LOG_ERR("H264EncStrmStart error=%d", ret); + return -EIO; + } + + output->bytesused = encOut.streamSize; + LOG_DBG("SPS/PPS generated, size= %d", output->bytesused); + + data->resync = true; + + return 0; +} + +static int encoder_end(struct stm32_venc_data *data) +{ + H264EncIn encIn = {0}; + H264EncOut encOut = {0}; + + if (data->encoder != NULL) { + H264EncStrmEnd(data->encoder, &encIn, &encOut); + data->encoder = NULL; + } + + return 0; +} + +static int encode_frame(struct stm32_venc_data *data) +{ + int ret = H264ENC_FRAME_READY; + struct video_buffer *input; + struct video_buffer *output; + H264EncIn encIn = {0}; + H264EncOut encOut = {0}; + + if (k_fifo_is_empty(&data->in_fifo_in) || k_fifo_is_empty(&data->out_fifo_in)) { + /* Encoding deferred to next buffer queueing */ + return 0; + } + + input = k_fifo_get(&data->in_fifo_in, K_NO_WAIT); + output = k_fifo_get(&data->out_fifo_in, K_NO_WAIT); + + if (data->encoder == NULL) { + ret = encoder_prepare(data); + if (ret) { + goto out; + } + + ret = encoder_start(data, output); + + /* + * Output buffer is now filled with SPS/PPS header, return it to application. + * Input buffer is dropped to not introduce latency. + */ + goto out; + } + + /* one key frame every seconds */ + if ((data->frame_nb % VENC_DEFAULT_FRAMERATE) == 0 || data->resync) { + /* if frame is the first or resync needed: set as intra coded */ + encIn.codingType = H264ENC_INTRA_FRAME; + } else { + /* if there was a frame previously, set as predicted */ + encIn.timeIncrement = 1; + encIn.codingType = H264ENC_PREDICTED_FRAME; + } + + encIn.ipf = H264ENC_REFERENCE_AND_REFRESH; + encIn.ltrf = H264ENC_REFERENCE; + + /* set input buffers to structures */ + encIn.busLuma = (ptr_t)input->buffer; + encIn.busChromaU = (ptr_t)encIn.busLuma + data->in_fmt.width * data->in_fmt.height; + + encIn.pOutBuf = (uint32_t *)output->buffer; + encIn.busOutBuf = (uint32_t)encIn.pOutBuf; + encIn.outBufSize = output->size; + encOut.streamSize = 0; + + ret = H264EncStrmEncode(data->encoder, &encIn, &encOut, NULL, NULL, NULL); + output->bytesused = encOut.streamSize; + LOG_DBG("output=%p, encOut.streamSize=%d", output, encOut.streamSize); + + switch (ret) { + case H264ENC_FRAME_READY: + /* save stream */ + if (encOut.streamSize == 0) { + /* Nothing encoded */ + data->resync = true; + goto out; + } + output->bytesused = encOut.streamSize; + break; + case H264ENC_FUSE_ERROR: + LOG_ERR("H264EncStrmEncode error=%d", ret); + + LOG_ERR("DCMIPP and VENC desync at frame %d, restart the video", data->frame_nb); + encoder_end(data); + + ret = encoder_start(data, output); + if (ret) { + goto out; + } + break; + default: + LOG_ERR("H264EncStrmEncode error=%d", ret); + LOG_ERR("error encoding frame %d", data->frame_nb); + + encoder_end(data); + + ret = encoder_start(data, output); + if (ret) { + goto out; + } + + data->resync = true; + + ret = -EIO; + goto out; + } + + data->frame_nb++; + +out: + k_fifo_put(&data->in_fifo_out, input); + k_fifo_put(&data->out_fifo_out, output); + + return ret; +} + +static int stm32_venc_set_stream(const struct device *dev, bool enable, enum video_buf_type type) +{ + struct stm32_venc_data *data = dev->data; + + ARG_UNUSED(type); + + if (!enable) { + /* Stop VENC */ + encoder_end(data); + } + + return 0; +} + +static int stm32_venc_enqueue(const struct device *dev, struct video_buffer *vbuf) +{ + struct stm32_venc_data *data = dev->data; + int ret = 0; + + k_mutex_lock(&data->lock, K_FOREVER); + + if (vbuf->type == VIDEO_BUF_TYPE_INPUT) { + k_fifo_put(&data->in_fifo_in, vbuf); + } else { + k_fifo_put(&data->out_fifo_in, vbuf); + } + + ret = encode_frame(data); + + k_mutex_unlock(&data->lock); + return ret; +} + +static int stm32_venc_dequeue(const struct device *dev, struct video_buffer **vbuf, + k_timeout_t timeout) +{ + struct stm32_venc_data *data = dev->data; + int ret = 0; + + k_mutex_lock(&data->lock, K_FOREVER); + + if ((*vbuf)->type == VIDEO_BUF_TYPE_INPUT) { + *vbuf = k_fifo_get(&data->in_fifo_out, timeout); + } else { + *vbuf = k_fifo_get(&data->out_fifo_out, timeout); + } + + if (*vbuf == NULL) { + ret = -EAGAIN; + goto out; + } + +out: + k_mutex_unlock(&data->lock); + return ret; +} + +ISR_DIRECT_DECLARE(stm32_venc_isr) +{ + struct stm32_venc_ewl *inst = &ewl_instance; + const struct stm32_venc_config *config = inst->config; + uint32_t hw_handshake_status = + IS_BIT_SET(sys_read32(config->reg + BASE_HEncInstantInput), LOW_LATENCY_HW_ITF_EN); + uint32_t irq_status = sys_read32(config->reg + BASE_HEncIRQ); + + inst->irq_status = irq_status; + inst->irq_cnt++; + + if (hw_handshake_status == 0 && (irq_status & ASIC_STATUS_FUSE) != 0) { + sys_write32(ASIC_STATUS_FUSE | ASIC_IRQ_LINE, config->reg + BASE_HEncIRQ); + /* read back the IRQ status to update its value */ + irq_status = sys_read32(config->reg + BASE_HEncIRQ); + } + + if (irq_status != 0U) { + /* status flag is raised, + * clear the ones that the IRQ needs to clear + * and signal to EWLWaitHwRdy + */ + sys_write32(ASIC_STATUS_SLICE_READY | ASIC_IRQ_LINE, config->reg + BASE_HEncIRQ); + } + + k_sem_give(&inst->complete); + + return 0; +} + +#define VENC_FORMAT_CAP(pixfmt) \ + { \ + .pixelformat = pixfmt, \ + .width_min = 48, \ + .width_max = 1920, \ + .height_min = 48, \ + .height_max = 1088, \ + .width_step = 16, \ + .height_step = 16, \ + } + +static const struct video_format_cap in_fmts[] = { + VENC_FORMAT_CAP(VIDEO_PIX_FMT_NV12), + VENC_FORMAT_CAP(VIDEO_PIX_FMT_RGB565), + {0}, +}; + +static const struct video_format_cap out_fmts[] = { + VENC_FORMAT_CAP(VIDEO_PIX_FMT_H264), + {0}, +}; + +static int stm32_venc_get_caps(const struct device *dev, struct video_caps *caps) +{ + if (caps->type == VIDEO_BUF_TYPE_INPUT) { + caps->format_caps = in_fmts; + } else { + caps->format_caps = out_fmts; + } + + /* VENC produces full frames */ + caps->min_line_count = caps->max_line_count = LINE_COUNT_HEIGHT; + caps->min_vbuf_count = 1; + + return 0; +} + +static DEVICE_API(video, stm32_venc_driver_api) = { + .set_format = stm32_venc_set_fmt, + .get_format = stm32_venc_get_fmt, + .set_stream = stm32_venc_set_stream, + .enqueue = stm32_venc_enqueue, + .dequeue = stm32_venc_dequeue, + .get_caps = stm32_venc_get_caps, +}; + +static void stm32_venc_irq_config_func(const struct device *dev) +{ + IRQ_DIRECT_CONNECT(DT_INST_IRQN(0), DT_INST_IRQ(0, priority), stm32_venc_isr, 0); + irq_enable(DT_INST_IRQN(0)); +} + +static struct stm32_venc_data stm32_venc_data_0 = {}; + +static const struct stm32_venc_config stm32_venc_config_0 = { + .reg = DT_INST_REG_ADDR(0), + .pclken = {.bus = DT_INST_CLOCKS_CELL(0, bus), .enr = DT_INST_CLOCKS_CELL(0, bits)}, + .reset = RESET_DT_SPEC_INST_GET_BY_IDX(0, 0), + .irq_config = stm32_venc_irq_config_func, +}; + +static void RISAF_Config(void) +{ + /* Define and initialize the master configuration structure */ + RIMC_MasterConfig_t RIMC_master = {0}; + + /* Enable the clock for the RIFSC (RIF Security Controller) */ + __HAL_RCC_RIFSC_CLK_ENABLE(); + + RIMC_master.MasterCID = RIF_CID_1; + RIMC_master.SecPriv = RIF_ATTRIBUTE_SEC | RIF_ATTRIBUTE_PRIV; + + /* Configure the master attributes for the video encoder peripheral (VENC) */ + HAL_RIF_RIMC_ConfigMasterAttributes(RIF_MASTER_INDEX_VENC, &RIMC_master); + + /* Set the secure and privileged attributes for the VENC as a slave */ + HAL_RIF_RISC_SetSlaveSecureAttributes(RIF_RISC_PERIPH_INDEX_VENC, + RIF_ATTRIBUTE_SEC | RIF_ATTRIBUTE_PRIV); +} + +static int stm32_venc_init(const struct device *dev) +{ + const struct stm32_venc_config *config = dev->config; + struct stm32_venc_data *data = dev->data; + int err; + + /* Enable VENC clock */ + err = stm32_venc_enable_clock(dev); + if (err < 0) { + LOG_ERR("clock enabling failed."); + return err; + } + + /* Reset VENC */ + if (!device_is_ready(config->reset.dev)) { + LOG_ERR("reset controller not ready"); + return -ENODEV; + } + reset_line_toggle_dt(&config->reset); + + data->dev = dev; + k_mutex_init(&data->lock); + k_fifo_init(&data->in_fifo_in); + k_fifo_init(&data->in_fifo_out); + k_fifo_init(&data->out_fifo_in); + k_fifo_init(&data->out_fifo_out); + + /* Run IRQ init */ + config->irq_config(dev); + + RISAF_Config(); + + LOG_DBG("CPU frequency : %d", HAL_RCC_GetCpuClockFreq() / 1000000); + LOG_DBG("sysclk frequency : %d", HAL_RCC_GetSysClockFreq() / 1000000); + LOG_DBG("pclk5 frequency : %d", HAL_RCC_GetPCLK5Freq() / 1000000); + + /* default input */ + data->in_fmt.width = VENC_DEFAULT_WIDTH; + data->in_fmt.height = VENC_DEFAULT_HEIGHT; + data->in_fmt.pixelformat = VENC_DEFAULT_IN_FMT; + data->in_fmt.pitch = data->in_fmt.width; + + /* default output */ + data->out_fmt.width = VENC_DEFAULT_WIDTH; + data->out_fmt.height = VENC_DEFAULT_HEIGHT; + data->out_fmt.pixelformat = VENC_DEFAULT_OUT_FMT; + + /* store config for register accesses */ + ewl_instance.config = config; + + LOG_DBG("%s inited", dev->name); + + return 0; +} + +DEVICE_DT_INST_DEFINE(0, &stm32_venc_init, NULL, &stm32_venc_data_0, &stm32_venc_config_0, + POST_KERNEL, CONFIG_VIDEO_INIT_PRIORITY, &stm32_venc_driver_api); diff --git a/tests/drivers/build_all/video/testcase.yaml b/tests/drivers/build_all/video/testcase.yaml index 30430a791ee11..dc8300b6e37a1 100644 --- a/tests/drivers/build_all/video/testcase.yaml +++ b/tests/drivers/build_all/video/testcase.yaml @@ -41,3 +41,6 @@ tests: - ek_ra8d1/r7fa8d1bhecbd extra_args: - platform:ek_ra81/r7fa8d1bhecbd:SHIELD="dvp_20pin_ov7670" + drivers.video.stm32_venc.build: + platform_allow: + - stm32n6570_dk/stm32n657xx/sb From 0055ab49306517dfcc5635dba1d2bcec7da9d9b3 Mon Sep 17 00:00:00 2001 From: Hugues Fruchet Date: Wed, 18 Jun 2025 11:48:07 +0200 Subject: [PATCH 6/7] dts: arm: st: n6: add venc node Add node describing the venc in stm32n6.dtsi Signed-off-by: Hugues Fruchet --- dts/arm/st/n6/stm32n6.dtsi | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dts/arm/st/n6/stm32n6.dtsi b/dts/arm/st/n6/stm32n6.dtsi index 21b39c8fa8987..990d6cd0cf02d 100644 --- a/dts/arm/st/n6/stm32n6.dtsi +++ b/dts/arm/st/n6/stm32n6.dtsi @@ -1299,6 +1299,15 @@ resets = <&rctl STM32_RESET(AHB5, 31)>; status = "disabled"; }; + + venc: venc@58005000 { + compatible = "st,stm32-venc"; + reg = <0x58005000 0x1000>; + interrupts = <62 0>; + clocks = <&rcc STM32_CLOCK(APB5, 5)>; + resets = <&rctl STM32_RESET(APB5, 5)>; + status = "disabled"; + }; }; }; From 490790696d598898e3ad881377327351efff2cf7 Mon Sep 17 00:00:00 2001 From: Erwan Gouriou Date: Wed, 1 Oct 2025 11:43:13 +0200 Subject: [PATCH 7/7] samples: video: tcpserversink: Add stm32n6570_dk as integration platform Required to build VIDEO_STM32_VENC. Signed-off-by: Erwan Gouriou Signed-off-by: Hugues Fruchet --- boards/st/stm32n6570_dk/twister.yaml | 1 + samples/drivers/video/tcpserversink/sample.yaml | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/boards/st/stm32n6570_dk/twister.yaml b/boards/st/stm32n6570_dk/twister.yaml index 1e365ef742776..fc6b07af5fe98 100644 --- a/boards/st/stm32n6570_dk/twister.yaml +++ b/boards/st/stm32n6570_dk/twister.yaml @@ -21,6 +21,7 @@ supported: - uart - usb_device - usbd + - video variants: stm32n6570_dk/stm32n657xx: twister: false diff --git a/samples/drivers/video/tcpserversink/sample.yaml b/samples/drivers/video/tcpserversink/sample.yaml index 9e9123b48c070..76ee4c3db8769 100644 --- a/samples/drivers/video/tcpserversink/sample.yaml +++ b/samples/drivers/video/tcpserversink/sample.yaml @@ -8,11 +8,16 @@ tests: - net - socket - shield - platform_allow: mimxrt1064_evk/mimxrt1064 + platform_allow: + - mimxrt1064_evk/mimxrt1064 + - stm32n6570_dk/stm32n657xx/sb depends_on: - video - netif integration_platforms: - mimxrt1064_evk/mimxrt1064 + - stm32n6570_dk/stm32n657xx/sb extra_args: - platform:mimxrt1064_evk/mimxrt1064:SHIELD=dvp_fpc24_mt9m114 + - platform:stm32n6570_dk/stm32n657xx/sb:SNIPPET=video-stm32-venc + - platform:stm32n6570_dk/stm32n657xx/sb:SHIELD=st_b_cams_imx_mb1854