From 736fe2aa246e81269cbcab6688ebc0a532a85288 Mon Sep 17 00:00:00 2001 From: James Bowley Date: Tue, 11 Jun 2019 09:57:20 +0100 Subject: [PATCH 1/9] Add missing codecs to cudacodec which uses Nvidia Video Codec SDK including checks to ensure codec used in input video file is supported on the current device. --- .../cudacodec/include/opencv2/cudacodec.hpp | 4 ++ modules/cudacodec/src/cuvid_video_source.cpp | 1 + modules/cudacodec/src/video_decoder.cpp | 41 ++++++++++++++----- 3 files changed, 35 insertions(+), 11 deletions(-) diff --git a/modules/cudacodec/include/opencv2/cudacodec.hpp b/modules/cudacodec/include/opencv2/cudacodec.hpp index e404a48ce9..dd8ae20007 100644 --- a/modules/cudacodec/include/opencv2/cudacodec.hpp +++ b/modules/cudacodec/include/opencv2/cudacodec.hpp @@ -250,6 +250,9 @@ enum Codec JPEG, H264_SVC, H264_MVC, + HEVC, + VP8, + VP9, Uncompressed_YUV420 = (('I'<<24)|('Y'<<16)|('U'<<8)|('V')), //!< Y,U,V (4:2:0) Uncompressed_YV12 = (('Y'<<24)|('V'<<16)|('1'<<8)|('2')), //!< Y,V,U (4:2:0) @@ -274,6 +277,7 @@ struct FormatInfo { Codec codec; ChromaFormat chromaFormat; + int nBitDepthMinus8; int width; int height; }; diff --git a/modules/cudacodec/src/cuvid_video_source.cpp b/modules/cudacodec/src/cuvid_video_source.cpp index fd6eac72de..75366a0f7a 100644 --- a/modules/cudacodec/src/cuvid_video_source.cpp +++ b/modules/cudacodec/src/cuvid_video_source.cpp @@ -70,6 +70,7 @@ cv::cudacodec::detail::CuvidVideoSource::CuvidVideoSource(const String& fname) format_.codec = static_cast(vidfmt.codec); format_.chromaFormat = static_cast(vidfmt.chroma_format); + format_.nBitDepthMinus8 = vidfmt.bit_depth_luma_minus8; format_.width = vidfmt.coded_width; format_.height = vidfmt.coded_height; } diff --git a/modules/cudacodec/src/video_decoder.cpp b/modules/cudacodec/src/video_decoder.cpp index 35919c3f56..ec508eac00 100644 --- a/modules/cudacodec/src/video_decoder.cpp +++ b/modules/cudacodec/src/video_decoder.cpp @@ -57,23 +57,42 @@ void cv::cudacodec::detail::VideoDecoder::create(const FormatInfo& videoFormat) cudaVideoCreate_PreferCUVID; // Validate video format. These are the currently supported formats via NVCUVID - CV_Assert(cudaVideoCodec_MPEG1 == _codec || - cudaVideoCodec_MPEG2 == _codec || - cudaVideoCodec_MPEG4 == _codec || - cudaVideoCodec_VC1 == _codec || - cudaVideoCodec_H264 == _codec || - cudaVideoCodec_JPEG == _codec || - cudaVideoCodec_YUV420== _codec || - cudaVideoCodec_YV12 == _codec || - cudaVideoCodec_NV12 == _codec || - cudaVideoCodec_YUYV == _codec || - cudaVideoCodec_UYVY == _codec ); + CV_Assert( cudaVideoCodec_MPEG1 == _codec || + cudaVideoCodec_MPEG2 == _codec || + cudaVideoCodec_MPEG4 == _codec || + cudaVideoCodec_VC1 == _codec || + cudaVideoCodec_H264 == _codec || + cudaVideoCodec_JPEG == _codec || + cudaVideoCodec_H264_SVC == _codec || + cudaVideoCodec_H264_MVC == _codec || + cudaVideoCodec_HEVC == _codec || + cudaVideoCodec_VP8 == _codec || + cudaVideoCodec_VP9 == _codec || + cudaVideoCodec_YUV420 == _codec || + cudaVideoCodec_YV12 == _codec || + cudaVideoCodec_NV12 == _codec || + cudaVideoCodec_YUYV == _codec || + cudaVideoCodec_UYVY == _codec); CV_Assert(cudaVideoChromaFormat_Monochrome == _chromaFormat || cudaVideoChromaFormat_420 == _chromaFormat || cudaVideoChromaFormat_422 == _chromaFormat || cudaVideoChromaFormat_444 == _chromaFormat); + // Check video format is supported by GPU's hardware video decoder + CUVIDDECODECAPS decodeCaps = {}; + decodeCaps.eCodecType = _codec; + decodeCaps.eChromaFormat = _chromaFormat; + decodeCaps.nBitDepthMinus8 = videoFormat.nBitDepthMinus8; + cuSafeCall(cuvidGetDecoderCaps(&decodeCaps)); + if (!decodeCaps.bIsSupported) + CV_Error(Error::StsUnsupportedFormat, "Video source is not supported by hardware video decoder"); + + CV_Assert(videoFormat.width >= decodeCaps.nMinWidth && + videoFormat.height >= decodeCaps.nMinHeight && + videoFormat.width <= decodeCaps.nMaxWidth && + videoFormat.height <= decodeCaps.nMaxHeight); + // Fill the decoder-create-info struct from the given video-format struct. std::memset(&createInfo_, 0, sizeof(CUVIDDECODECREATEINFO)); From 4d9b036e5fe1597ff797521d7840d42f6d671629 Mon Sep 17 00:00:00 2001 From: cudawarped Date: Fri, 29 Oct 2021 18:37:40 +0100 Subject: [PATCH 2/9] Update cudacoded to 1) automatically write the raw encoded video stream to a video file and, 2) use less GPU memory by more closely mirroring the Nvidia samples. Specifically querying the decoder for the number of decode surfaces (h265 commonly uses 4) instead of always using 20 and not using adaptive deinterlacing when the video sequence is progressive. Additional updates to mirror the Nvidia sample include initializing the decoder so that HandleVideoSequence() gets called every time before the decoder is initialized, ensuring all the parameters for the decoder are provided by nvcudec. Added facility to decode AV1, not tested as VideoCapture doesn't return a valid fourcc for this. Add facility to decode MPEG4 video - requires modification to VideoCapture see pull request. --- .../cudacodec/include/opencv2/cudacodec.hpp | 67 +++++++++++++++++-- modules/cudacodec/src/cuvid_video_source.cpp | 7 +- modules/cudacodec/src/cuvid_video_source.hpp | 3 +- modules/cudacodec/src/ffmpeg_video_source.cpp | 65 ++++++++++++++++-- modules/cudacodec/src/ffmpeg_video_source.hpp | 15 ++++- modules/cudacodec/src/frame_queue.cpp | 28 ++++---- modules/cudacodec/src/frame_queue.hpp | 18 +++-- modules/cudacodec/src/video_decoder.cpp | 46 ++++++------- modules/cudacodec/src/video_decoder.hpp | 32 ++++----- modules/cudacodec/src/video_parser.cpp | 36 ++++++++-- modules/cudacodec/src/video_reader.cpp | 23 ++++--- modules/cudacodec/src/video_source.cpp | 9 ++- modules/cudacodec/src/video_source.hpp | 6 +- modules/cudacodec/test/test_video.cpp | 56 +++++++++++++++- 14 files changed, 309 insertions(+), 102 deletions(-) diff --git a/modules/cudacodec/include/opencv2/cudacodec.hpp b/modules/cudacodec/include/opencv2/cudacodec.hpp index b910fe951a..f98fa817d9 100644 --- a/modules/cudacodec/include/opencv2/cudacodec.hpp +++ b/modules/cudacodec/include/opencv2/cudacodec.hpp @@ -231,7 +231,7 @@ CV_EXPORTS_W Ptr createVideoWriter(const Ptr createVideoReader(const String& filename); +CV_EXPORTS_W Ptr createVideoReader(const String& filename, const String filenameToWrite = "", const bool autoDetectExt = false); /** @overload @param source RAW video source implemented by user. */ diff --git a/modules/cudacodec/src/cuvid_video_source.cpp b/modules/cudacodec/src/cuvid_video_source.cpp index aab7889352..72b72f37dd 100644 --- a/modules/cudacodec/src/cuvid_video_source.cpp +++ b/modules/cudacodec/src/cuvid_video_source.cpp @@ -76,6 +76,8 @@ cv::cudacodec::detail::CuvidVideoSource::CuvidVideoSource(const String& fname) format_.height = vidfmt.coded_height; format_.displayArea = Rect(Point(vidfmt.display_area.left, vidfmt.display_area.top), Point(vidfmt.display_area.right, vidfmt.display_area.bottom)); format_.valid = true; + if (vidfmt.frame_rate.numerator != 0 && vidfmt.frame_rate.denominator != 0) + format_.fps = vidfmt.frame_rate.numerator / (double)vidfmt.frame_rate.denominator; } cv::cudacodec::detail::CuvidVideoSource::~CuvidVideoSource() @@ -88,10 +90,9 @@ FormatInfo cv::cudacodec::detail::CuvidVideoSource::format() const return format_; } -void cv::cudacodec::detail::CuvidVideoSource::updateFormat(const int codedWidth, const int codedHeight) +void cv::cudacodec::detail::CuvidVideoSource::updateFormat(const FormatInfo& videoFormat) { - format_.width = codedWidth; - format_.height = codedHeight; + format_ = videoFormat; format_.valid = true; } diff --git a/modules/cudacodec/src/cuvid_video_source.hpp b/modules/cudacodec/src/cuvid_video_source.hpp index 314f3104cc..fddea2d7a3 100644 --- a/modules/cudacodec/src/cuvid_video_source.hpp +++ b/modules/cudacodec/src/cuvid_video_source.hpp @@ -55,7 +55,8 @@ class CuvidVideoSource : public VideoSource ~CuvidVideoSource(); FormatInfo format() const CV_OVERRIDE; - void updateFormat(const int codedWidth, const int codedHeight); + void updateFormat(const FormatInfo& videoFormat) CV_OVERRIDE; + void writeToFile(const std::string filename, const bool autoDetectExt = false) CV_OVERRIDE {}; void start() CV_OVERRIDE; void stop() CV_OVERRIDE; bool isStarted() const CV_OVERRIDE; diff --git a/modules/cudacodec/src/ffmpeg_video_source.cpp b/modules/cudacodec/src/ffmpeg_video_source.cpp index 019ac84b17..1c381938f9 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.cpp +++ b/modules/cudacodec/src/ffmpeg_video_source.cpp @@ -75,6 +75,7 @@ Codec FourccToCodec(int codec) case CV_FOURCC_MACRO('M', 'P', 'G', '1'): return MPEG1; case CV_FOURCC_MACRO('M', 'P', 'G', '2'): return MPEG2; case CV_FOURCC_MACRO('X', 'V', 'I', 'D'): // fallthru + case CV_FOURCC_MACRO('m', 'p', '4', 'v'): // fallthru case CV_FOURCC_MACRO('D', 'I', 'V', 'X'): return MPEG4; case CV_FOURCC_MACRO('W', 'V', 'C', '1'): return VC1; case CV_FOURCC_MACRO('H', '2', '6', '4'): // fallthru @@ -112,7 +113,18 @@ void FourccToChromaFormat(const int pixelFormat, ChromaFormat &chromaFormat, int } } -cv::cudacodec::detail::FFmpegVideoSource::FFmpegVideoSource(const String& fname) +static +std::string CodecToFileExtension(const Codec codec){ + switch (codec) { + case(Codec::H264): return ".h264"; + case(Codec::HEVC): return ".h265"; + case(Codec::VP8): return ".vp8"; + case(Codec::VP9): return ".vp9"; + default: return ""; + } +} + +cv::cudacodec::detail::FFmpegVideoSource::FFmpegVideoSource(const String& fname, const String& filenameToWrite, const bool autoDetectExt) { if (!videoio_registry::hasBackend(CAP_FFMPEG)) CV_Error(Error::StsNotImplemented, "FFmpeg backend not found"); @@ -125,6 +137,9 @@ cv::cudacodec::detail::FFmpegVideoSource::FFmpegVideoSource(const String& fname) CV_Error(Error::StsUnsupportedFormat, "Fetching of RAW video streams is not supported"); CV_Assert(cap.get(CAP_PROP_FORMAT) == -1); + if (!filenameToWrite.empty()) + writeToFile(filenameToWrite, autoDetectExt); + int codec = (int)cap.get(CAP_PROP_FOURCC); int pixelFormat = (int)cap.get(CAP_PROP_CODEC_PIXEL_FORMAT); @@ -133,6 +148,7 @@ cv::cudacodec::detail::FFmpegVideoSource::FFmpegVideoSource(const String& fname) format_.width = cap.get(CAP_PROP_FRAME_WIDTH); format_.displayArea = Rect(0, 0, format_.width, format_.height); format_.valid = false; + format_.fps = cap.get(CAP_PROP_FPS); FourccToChromaFormat(pixelFormat, format_.chromaFormat, format_.nBitDepthMinus8); } @@ -147,18 +163,59 @@ FormatInfo cv::cudacodec::detail::FFmpegVideoSource::format() const return format_; } -void cv::cudacodec::detail::FFmpegVideoSource::updateFormat(const int codedWidth, const int codedHeight) +void cv::cudacodec::detail::FFmpegVideoSource::updateFormat(const FormatInfo& videoFormat) { - format_.width = codedWidth; - format_.height = codedHeight; + format_ = videoFormat; format_.valid = true; } +void cv::cudacodec::detail::FFmpegVideoSource::writeToFile(const std::string _filename, const bool _autoDetectExt) { + std::lock_guard lck(mtx); + fileName = _filename; + if(fileName.empty()){ + if (file.is_open()) + file.close(); + restartRtspFileWrite = false; + return; + } + autoDetectExt = _autoDetectExt; + restartRtspFileWrite = true; +} + bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** data, size_t* size) { + std::lock_guard lck(mtx); cap >> rawFrame; *data = rawFrame.data; *size = rawFrame.total(); + + // always grab param sets if available in case writeToFile is called after construction + const int paramSetLen = cap.get(CAP_PROP_LF_PARAM_SET_LEN); + if (paramSetLen) { + parameterSets = Mat(1, paramSetLen, CV_8UC1); + memcpy(parameterSets.data, *data, paramSetLen); + } + + if (restartRtspFileWrite && cap.get(CAP_PROP_LF_KEY_FRAME)) { + if (file.is_open()) + file.close(); + if (autoDetectExt) + fileName += CodecToFileExtension(format_.codec); + file.open(fileName, std::ios::binary); + if (!file.is_open()) + return false; + restartRtspFileWrite = false; + if (!paramSetLen && !parameterSets.empty()) + writeParameterSets = true; + } + + if (file.is_open()) { + if (writeParameterSets) { + writeParameterSets = false; + file.write((char*)parameterSets.data, parameterSets.total()); + } + file.write((char*)*data, *size); + } return *size != 0; } diff --git a/modules/cudacodec/src/ffmpeg_video_source.hpp b/modules/cudacodec/src/ffmpeg_video_source.hpp index 0c802d80f0..7401baeca1 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.hpp +++ b/modules/cudacodec/src/ffmpeg_video_source.hpp @@ -44,6 +44,8 @@ #ifndef __FFMPEG_VIDEO_SOURCE_HPP__ #define __FFMPEG_VIDEO_SOURCE_HPP__ +#include + #include "opencv2/cudacodec.hpp" namespace cv { namespace cudacodec { namespace detail { @@ -51,20 +53,27 @@ namespace cv { namespace cudacodec { namespace detail { class FFmpegVideoSource : public RawVideoSource { public: - FFmpegVideoSource(const String& fname); + FFmpegVideoSource(const String& fname, const String& filenameToWrite, const bool autoDetectExt = false); ~FFmpegVideoSource(); bool getNextPacket(unsigned char** data, size_t* size) CV_OVERRIDE; FormatInfo format() const CV_OVERRIDE; - void updateFormat(const int codedWidth, const int codedHeight); + void updateFormat(const FormatInfo& videoFormat) CV_OVERRIDE; + void writeToFile(const std::string filename, const bool autoDetectExt = false) CV_OVERRIDE; private: FormatInfo format_; VideoCapture cap; - Mat rawFrame; + Mat rawFrame, parameterSets; + std::string fileName; + std::ofstream file; + bool autoDetectExt = false; + bool restartRtspFileWrite = false; + bool writeParameterSets = false; + std::mutex mtx; }; }}} diff --git a/modules/cudacodec/src/frame_queue.cpp b/modules/cudacodec/src/frame_queue.cpp index d3c42c902c..87fb1e7216 100644 --- a/modules/cudacodec/src/frame_queue.cpp +++ b/modules/cudacodec/src/frame_queue.cpp @@ -45,13 +45,17 @@ #ifdef HAVE_NVCUVID -cv::cudacodec::detail::FrameQueue::FrameQueue() : - endOfDecode_(0), - framesInQueue_(0), - readPosition_(0) -{ - std::memset(displayQueue_, 0, sizeof(displayQueue_)); - std::memset((void*) isFrameInUse_, 0, sizeof(isFrameInUse_)); +cv::cudacodec::detail::FrameQueue::~FrameQueue() { + if (isFrameInUse_) + delete[] isFrameInUse_; +} + +void cv::cudacodec::detail::FrameQueue::init(const int _maxSz) { + AutoLock autoLock(mtx_); + maxSz = _maxSz; + displayQueue_ = std::vector(maxSz, CUVIDPARSERDISPINFO()); + isFrameInUse_ = new volatile int[maxSz]; + std::memset((void*)isFrameInUse_, 0, sizeof(*isFrameInUse_) * maxSz); } bool cv::cudacodec::detail::FrameQueue::waitUntilFrameAvailable(int pictureIndex) @@ -82,10 +86,10 @@ void cv::cudacodec::detail::FrameQueue::enqueue(const CUVIDPARSERDISPINFO* picPa { AutoLock autoLock(mtx_); - if (framesInQueue_ < MaximumSize) + if (framesInQueue_ < maxSz) { - int writePosition = (readPosition_ + framesInQueue_) % MaximumSize; - displayQueue_[writePosition] = *picParams; + int writePosition = (readPosition_ + framesInQueue_) % maxSz; + displayQueue_.at(writePosition) = *picParams; framesInQueue_++; isFramePlaced = true; } @@ -106,8 +110,8 @@ bool cv::cudacodec::detail::FrameQueue::dequeue(CUVIDPARSERDISPINFO& displayInfo if (framesInQueue_ > 0) { int entry = readPosition_; - displayInfo = displayQueue_[entry]; - readPosition_ = (entry + 1) % MaximumSize; + displayInfo = displayQueue_.at(entry); + readPosition_ = (entry + 1) % maxSz; framesInQueue_--; return true; } diff --git a/modules/cudacodec/src/frame_queue.hpp b/modules/cudacodec/src/frame_queue.hpp index 3051a1e450..595c4fc17d 100644 --- a/modules/cudacodec/src/frame_queue.hpp +++ b/modules/cudacodec/src/frame_queue.hpp @@ -51,9 +51,8 @@ namespace cv { namespace cudacodec { namespace detail { class FrameQueue { public: - static const int MaximumSize = 20; // MAX_FRM_CNT; - - FrameQueue(); + ~FrameQueue(); + void init(const int _maxSz); void endDecode() { endOfDecode_ = true; } bool isEndOfDecode() const { return endOfDecode_ != 0;} @@ -80,13 +79,12 @@ class FrameQueue bool isInUse(int pictureIndex) const { return isFrameInUse_[pictureIndex] != 0; } Mutex mtx_; - - volatile int isFrameInUse_[MaximumSize]; - volatile int endOfDecode_; - - int framesInQueue_; - int readPosition_; - CUVIDPARSERDISPINFO displayQueue_[MaximumSize]; + volatile int* isFrameInUse_ = 0; + volatile int endOfDecode_ = 0; + int framesInQueue_ = 0; + int readPosition_ = 0; + std::vector< CUVIDPARSERDISPINFO> displayQueue_; + int maxSz = 0; }; }}} diff --git a/modules/cudacodec/src/video_decoder.cpp b/modules/cudacodec/src/video_decoder.cpp index 1293055bff..e5e9353e4f 100644 --- a/modules/cudacodec/src/video_decoder.cpp +++ b/modules/cudacodec/src/video_decoder.cpp @@ -47,13 +47,13 @@ void cv::cudacodec::detail::VideoDecoder::create(const FormatInfo& videoFormat) { - if (videoFormat.nBitDepthMinus8 > 0 || videoFormat.chromaFormat != YUV420) - CV_Error(Error::StsUnsupportedFormat, "NV12 output requires 8 bit YUV420"); + if (videoFormat.nBitDepthMinus8 > 0 || videoFormat.chromaFormat == YUV444) + CV_Error(Error::StsUnsupportedFormat, "NV12 output currently supported for 8 bit YUV420, YUV422 and Monochrome inputs."); - cudaVideoCodec _codec = static_cast(videoFormat.codec); - cudaVideoChromaFormat _chromaFormat = static_cast(videoFormat.chromaFormat); - - cudaVideoCreateFlags videoCreateFlags = (_codec == cudaVideoCodec_JPEG || _codec == cudaVideoCodec_MPEG2) ? + videoFormat_ = videoFormat; + const cudaVideoCodec _codec = static_cast(videoFormat.codec); + const cudaVideoChromaFormat _chromaFormat = static_cast(videoFormat.chromaFormat); + const cudaVideoCreateFlags videoCreateFlags = (_codec == cudaVideoCodec_JPEG || _codec == cudaVideoCodec_MPEG2) ? cudaVideoCreate_PreferCUDA : cudaVideoCreate_PreferCUVID; @@ -101,35 +101,29 @@ void cv::cudacodec::detail::VideoDecoder::create(const FormatInfo& videoFormat) if (!decodeCaps.bIsSupported) CV_Error(Error::StsUnsupportedFormat, "Video source is not supported by hardware video decoder"); - CV_Assert(videoFormat.width >= decodeCaps.nMinWidth && - videoFormat.height >= decodeCaps.nMinHeight && - videoFormat.width <= decodeCaps.nMaxWidth && - videoFormat.height <= decodeCaps.nMaxHeight); + CV_Assert(videoFormat.ulWidth >= decodeCaps.nMinWidth && + videoFormat.ulHeight >= decodeCaps.nMinHeight && + videoFormat.ulWidth <= decodeCaps.nMaxWidth && + videoFormat.ulHeight <= decodeCaps.nMaxHeight); CV_Assert((videoFormat.width >> 4)* (videoFormat.height >> 4) <= decodeCaps.nMaxMBCount); #endif - - // Fill the decoder-create-info struct from the given video-format struct. - std::memset(&createInfo_, 0, sizeof(CUVIDDECODECREATEINFO)); - // Create video decoder + CUVIDDECODECREATEINFO createInfo_ = {}; createInfo_.CodecType = _codec; - createInfo_.ulWidth = videoFormat.width; - createInfo_.ulHeight = videoFormat.height; - createInfo_.ulNumDecodeSurfaces = FrameQueue::MaximumSize; + createInfo_.ulWidth = videoFormat.ulWidth; + createInfo_.ulHeight = videoFormat.ulHeight; + createInfo_.ulNumDecodeSurfaces = videoFormat.ulNumDecodeSurfaces; createInfo_.ChromaFormat = _chromaFormat; createInfo_.OutputFormat = cudaVideoSurfaceFormat_NV12; - createInfo_.DeinterlaceMode = cudaVideoDeinterlaceMode_Adaptive; - - // No scaling - static const int MAX_FRAME_COUNT = 2; - - createInfo_.ulTargetWidth = createInfo_.ulWidth; - createInfo_.ulTargetHeight = createInfo_.ulHeight; - createInfo_.ulNumOutputSurfaces = MAX_FRAME_COUNT; // We won't simultaneously map more than 8 surfaces + createInfo_.DeinterlaceMode = static_cast(videoFormat.deinterlaceMode); + createInfo_.ulTargetWidth = videoFormat.width; + createInfo_.ulTargetHeight = videoFormat.height; + createInfo_.ulMaxWidth = videoFormat.ulMaxWidth; + createInfo_.ulMaxHeight = videoFormat.ulMaxHeight; + createInfo_.ulNumOutputSurfaces = 2; createInfo_.ulCreationFlags = videoCreateFlags; createInfo_.vidLock = lock_; - cuSafeCall(cuCtxPushCurrent(ctx_)); cuSafeCall(cuvidCreateDecoder(&decoder_, &createInfo_)); cuSafeCall(cuCtxPopCurrent(NULL)); diff --git a/modules/cudacodec/src/video_decoder.hpp b/modules/cudacodec/src/video_decoder.hpp index 4e1da2472d..452d16171e 100644 --- a/modules/cudacodec/src/video_decoder.hpp +++ b/modules/cudacodec/src/video_decoder.hpp @@ -49,9 +49,9 @@ namespace cv { namespace cudacodec { namespace detail { class VideoDecoder { public: - VideoDecoder(const FormatInfo& videoFormat, CUcontext ctx, CUvideoctxlock lock) : ctx_(ctx), lock_(lock), decoder_(0) + VideoDecoder(const Codec& codec, CUcontext ctx, CUvideoctxlock lock) : ctx_(ctx), lock_(lock), decoder_(0) { - create(videoFormat); + videoFormat_.codec = codec; } ~VideoDecoder() @@ -63,17 +63,18 @@ class VideoDecoder void release(); // Get the code-type currently used. - cudaVideoCodec codec() const { return createInfo_.CodecType; } - unsigned long maxDecodeSurfaces() const { return createInfo_.ulNumDecodeSurfaces; } + cudaVideoCodec codec() const { return static_cast(videoFormat_.codec); } + unsigned long maxDecodeSurfaces() const { return videoFormat_.ulNumDecodeSurfaces; } - unsigned long frameWidth() const { return createInfo_.ulWidth; } - unsigned long frameHeight() const { return createInfo_.ulHeight; } + unsigned long frameWidth() const { return videoFormat_.ulWidth; } + unsigned long frameHeight() const { return videoFormat_.ulHeight; } + FormatInfo format() { AutoLock autoLock(mtx_); return videoFormat_;} - unsigned long targetWidth() const { return createInfo_.ulTargetWidth; } - unsigned long targetHeight() const { return createInfo_.ulTargetHeight; } + unsigned long targetWidth() { return videoFormat_.width; } + unsigned long targetHeight() { return videoFormat_.height; } - cudaVideoChromaFormat chromaFormat() const { return createInfo_.ChromaFormat; } - int nBitDepthMinus8() const { return createInfo_.bitDepthMinus8; } + cudaVideoChromaFormat chromaFormat() const { return static_cast(videoFormat_.chromaFormat); } + int nBitDepthMinus8() const { return videoFormat_.nBitDepthMinus8; } bool decodePicture(CUVIDPICPARAMS* picParams) { @@ -87,8 +88,7 @@ class VideoDecoder cuSafeCall( cuvidMapVideoFrame(decoder_, picIdx, &ptr, &pitch, &videoProcParams) ); - - return cuda::GpuMat(targetHeight() * 3 / 2, targetWidth(), CV_8UC1, (void*) ptr, pitch); + return cuda::GpuMat(frameHeight() * 3 / 2, frameWidth(), CV_8UC1, (void*) ptr, pitch); } void unmapFrame(cuda::GpuMat& frame) @@ -98,10 +98,12 @@ class VideoDecoder } private: + //Codec codec_ = NumCodecs; + CUcontext ctx_ = 0; CUvideoctxlock lock_; - CUcontext ctx_; - CUVIDDECODECREATEINFO createInfo_; - CUvideodecoder decoder_; + CUvideodecoder decoder_ = 0; + FormatInfo videoFormat_ = {}; + Mutex mtx_; }; }}} diff --git a/modules/cudacodec/src/video_parser.cpp b/modules/cudacodec/src/video_parser.cpp index 5327f22a93..907ea2d7e7 100644 --- a/modules/cudacodec/src/video_parser.cpp +++ b/modules/cudacodec/src/video_parser.cpp @@ -52,7 +52,7 @@ cv::cudacodec::detail::VideoParser::VideoParser(VideoDecoder* videoDecoder, Fram std::memset(¶ms, 0, sizeof(CUVIDPARSERPARAMS)); params.CodecType = videoDecoder->codec(); - params.ulMaxNumDecodeSurfaces = videoDecoder->maxDecodeSurfaces(); + params.ulMaxNumDecodeSurfaces = 1; params.ulMaxDisplayDelay = 1; // this flag is needed so the parser will push frames out to the decoder as quickly as it can params.pUserData = this; params.pfnSequenceCallback = HandleVideoSequence; // Called before decoding frames and/or whenever there is a format change @@ -80,7 +80,7 @@ bool cv::cudacodec::detail::VideoParser::parseVideoData(const unsigned char* dat return false; } - const int maxUnparsedPackets = 20; + constexpr int maxUnparsedPackets = 20; ++unparsedPackets_; if (unparsedPackets_ > maxUnparsedPackets) @@ -106,17 +106,39 @@ int CUDAAPI cv::cudacodec::detail::VideoParser::HandleVideoSequence(void* userDa format->coded_width != thiz->videoDecoder_->frameWidth() || format->coded_height != thiz->videoDecoder_->frameHeight() || format->chroma_format != thiz->videoDecoder_->chromaFormat()|| - format->bit_depth_luma_minus8 != thiz->videoDecoder_->nBitDepthMinus8()) + format->bit_depth_luma_minus8 != thiz->videoDecoder_->nBitDepthMinus8() || + format->min_num_decode_surfaces != thiz->videoDecoder_->maxDecodeSurfaces()) { FormatInfo newFormat; - newFormat.codec = static_cast(format->codec); newFormat.chromaFormat = static_cast(format->chroma_format); + newFormat.nBitDepthMinus8 = format->bit_depth_luma_minus8; + newFormat.ulWidth = format->coded_width; + newFormat.ulHeight = format->coded_height; newFormat.width = format->coded_width; newFormat.height = format->coded_height; newFormat.displayArea = Rect(Point(format->display_area.left, format->display_area.top), Point(format->display_area.right, format->display_area.bottom)); - newFormat.nBitDepthMinus8 = format->bit_depth_luma_minus8; - + newFormat.fps = format->frame_rate.numerator / static_cast(format->frame_rate.denominator); + newFormat.ulNumDecodeSurfaces = format->min_num_decode_surfaces; + if (format->progressive_sequence) + newFormat.deinterlaceMode = Weave; + else + newFormat.deinterlaceMode = Adaptive; + int maxW = 0, maxH = 0; + // AV1 has max width/height of sequence in sequence header + if (format->codec == cudaVideoCodec_AV1 && format->seqhdr_data_length > 0) + { + CUVIDEOFORMATEX* vidFormatEx = (CUVIDEOFORMATEX*)format; + maxW = vidFormatEx->av1.max_width; + maxH = vidFormatEx->av1.max_height; + } + if (maxW < (int)format->coded_width) + maxW = format->coded_width; + if (maxH < (int)format->coded_height) + maxH = format->coded_height; + newFormat.ulMaxWidth = maxW; + newFormat.ulMaxHeight = maxH; + thiz->frameQueue_->init(newFormat.ulNumDecodeSurfaces); try { thiz->videoDecoder_->release(); @@ -129,7 +151,7 @@ int CUDAAPI cv::cudacodec::detail::VideoParser::HandleVideoSequence(void* userDa } } - return true; + return thiz->videoDecoder_->maxDecodeSurfaces(); } int CUDAAPI cv::cudacodec::detail::VideoParser::HandlePictureDecode(void* userData, CUVIDPICPARAMS* picParams) diff --git a/modules/cudacodec/src/video_reader.cpp b/modules/cudacodec/src/video_reader.cpp index 51f49225c2..b705e1618d 100644 --- a/modules/cudacodec/src/video_reader.cpp +++ b/modules/cudacodec/src/video_reader.cpp @@ -48,7 +48,7 @@ using namespace cv::cudacodec; #ifndef HAVE_NVCUVID -Ptr cv::cudacodec::createVideoReader(const String&) { throw_no_cuda(); return Ptr(); } +Ptr cv::cudacodec::createVideoReader(const String&, const String = "", const bool = false) { throw_no_cuda(); return Ptr(); } Ptr cv::cudacodec::createVideoReader(const Ptr&) { throw_no_cuda(); return Ptr(); } #else // HAVE_NVCUVID @@ -69,10 +69,12 @@ namespace FormatInfo format() const CV_OVERRIDE; + void writeToFile(const std::string filename, const bool autoDetectExt = false) CV_OVERRIDE; + private: Ptr videoSource_; - Ptr frameQueue_; + Ptr frameQueue_ = 0; Ptr videoDecoder_; Ptr videoParser_; @@ -97,11 +99,9 @@ namespace CUcontext ctx; cuSafeCall( cuCtxGetCurrent(&ctx) ); cuSafeCall( cuvidCtxLockCreate(&lock_, ctx) ); - - frameQueue_.reset(new FrameQueue); - videoDecoder_.reset(new VideoDecoder(videoSource_->format(), ctx, lock_)); + frameQueue_.reset(new FrameQueue()); + videoDecoder_.reset(new VideoDecoder(videoSource_->format().codec, ctx, lock_)); videoParser_.reset(new VideoParser(videoDecoder_, frameQueue_)); - videoSource_->setVideoParser(videoParser_); videoSource_->start(); } @@ -148,7 +148,7 @@ namespace bool isProgressive = displayInfo.progressive_frame != 0; const int num_fields = isProgressive ? 1 : 2 + displayInfo.repeat_first_field; - videoSource_->updateFormat(videoDecoder_->targetWidth(), videoDecoder_->targetHeight()); + videoSource_->updateFormat(videoDecoder_->format()); for (int active_field = 0; active_field < num_fields; ++active_field) { @@ -192,9 +192,14 @@ namespace return true; } + + void VideoReaderImpl::writeToFile(const std::string filename, const bool autoDetectExt) + { + return videoSource_->writeToFile(filename, autoDetectExt); + } } -Ptr cv::cudacodec::createVideoReader(const String& filename) +Ptr cv::cudacodec::createVideoReader(const String& filename, const String filenameToWrite, const bool autoDetectExt) { CV_Assert( !filename.empty() ); @@ -203,7 +208,7 @@ Ptr cv::cudacodec::createVideoReader(const String& filename) try { // prefer ffmpeg to cuvidGetSourceVideoFormat() which doesn't always return the corrct raw pixel format - Ptr source(new FFmpegVideoSource(filename)); + Ptr source(new FFmpegVideoSource(filename, filenameToWrite, autoDetectExt)); videoSource.reset(new RawVideoSourceWrapper(source)); } catch (...) diff --git a/modules/cudacodec/src/video_source.cpp b/modules/cudacodec/src/video_source.cpp index 3a233b9629..36e1350f31 100644 --- a/modules/cudacodec/src/video_source.cpp +++ b/modules/cudacodec/src/video_source.cpp @@ -65,9 +65,14 @@ cv::cudacodec::FormatInfo cv::cudacodec::detail::RawVideoSourceWrapper::format() return source_->format(); } -void cv::cudacodec::detail::RawVideoSourceWrapper::updateFormat(const int codedWidth, const int codedHeight) +void cv::cudacodec::detail::RawVideoSourceWrapper::updateFormat(const FormatInfo& videoFormat) { - source_->updateFormat(codedWidth,codedHeight); + source_->updateFormat(videoFormat); +} + +void cv::cudacodec::detail::RawVideoSourceWrapper::writeToFile(const std::string filename, const bool autoDetectExt) +{ + source_->writeToFile(filename, autoDetectExt); } void cv::cudacodec::detail::RawVideoSourceWrapper::start() diff --git a/modules/cudacodec/src/video_source.hpp b/modules/cudacodec/src/video_source.hpp index f17d4ab080..125156efdc 100644 --- a/modules/cudacodec/src/video_source.hpp +++ b/modules/cudacodec/src/video_source.hpp @@ -56,7 +56,8 @@ class VideoSource virtual ~VideoSource() {} virtual FormatInfo format() const = 0; - virtual void updateFormat(const int codedWidth, const int codedHeight) = 0; + virtual void updateFormat(const FormatInfo& videoFormat) = 0; + virtual void writeToFile(const std::string filename, const bool autoDetectExt) = 0; virtual void start() = 0; virtual void stop() = 0; virtual bool isStarted() const = 0; @@ -77,7 +78,8 @@ class RawVideoSourceWrapper : public VideoSource RawVideoSourceWrapper(const Ptr& source); FormatInfo format() const CV_OVERRIDE; - void updateFormat(const int codedWidth, const int codedHeight) CV_OVERRIDE; + void updateFormat(const FormatInfo& videoFormat) CV_OVERRIDE; + void writeToFile(const std::string filename, const bool autoDetectExt = false) CV_OVERRIDE; void start() CV_OVERRIDE; void stop() CV_OVERRIDE; bool isStarted() const CV_OVERRIDE; diff --git a/modules/cudacodec/test/test_video.cpp b/modules/cudacodec/test/test_video.cpp index 31a001f9d1..88a0495740 100644 --- a/modules/cudacodec/test/test_video.cpp +++ b/modules/cudacodec/test/test_video.cpp @@ -41,6 +41,7 @@ //M*/ #include "test_precomp.hpp" +#include "opencv2/cudaarithm.hpp" namespace opencv_test { namespace { @@ -49,6 +50,10 @@ PARAM_TEST_CASE(Video, cv::cuda::DeviceInfo, std::string) { }; +PARAM_TEST_CASE(VideoReadWrite, cv::cuda::DeviceInfo, std::string) +{ +}; + #if defined(HAVE_NVCUVID) ////////////////////////////////////////////////////// // VideoReader @@ -74,6 +79,47 @@ CUDA_TEST_P(Video, Reader) ASSERT_FALSE(frame.empty()); } } + +CUDA_TEST_P(VideoReadWrite, Reader) +{ + cv::cuda::setDevice(GET_PARAM(0).deviceID()); + + // RTSP streaming is only supported by the FFmpeg back end + if (!videoio_registry::hasBackend(CAP_FFMPEG)) + throw SkipTestException("FFmpeg backend not found"); + + std::string inputFile = std::string(cvtest::TS::ptr()->get_data_path()) + "../" + GET_PARAM(1); + const string fileNameOut = tempfile("test_container_stream"); + { + cv::Ptr reader = cv::cudacodec::createVideoReader(inputFile, fileNameOut); + cv::cuda::GpuMat frame; + for (int i = 0; i < 100; i++) + { + reader->writeToFile(fileNameOut.c_str()); + ASSERT_TRUE(reader->nextFrame(frame)); + ASSERT_FALSE(frame.empty()); + } + } + + std::cout << "Checking written video stream: " << fileNameOut << std::endl; + + { + cv::Ptr readerReference = cv::cudacodec::createVideoReader(inputFile); + cv::Ptr readerActual = cv::cudacodec::createVideoReader(fileNameOut); + cv::cuda::GpuMat reference, actual; + cv::Mat referenceHost, actualHost; + for (int i = 0; i < 100; i++) + { + ASSERT_TRUE(readerReference->nextFrame(reference)); + ASSERT_TRUE(readerActual->nextFrame(actual)); + actual.download(actualHost); + reference.download(referenceHost); + ASSERT_TRUE(cvtest::norm(actualHost, referenceHost, NORM_INF) == 0); + } + } + + ASSERT_EQ(0, remove(fileNameOut.c_str())); +} #endif // HAVE_NVCUVID #if defined(_WIN32) && defined(HAVE_NVCUVENC) @@ -125,11 +171,17 @@ CUDA_TEST_P(Video, Writer) #endif // _WIN32, HAVE_NVCUVENC -#define VIDEO_SRC "cv/video/768x576.avi", "cv/video/1920x1080.avi", "highgui/video/big_buck_bunny.avi", \ +#define VIDEO_SRC_R "cv/video/768x576.avi", "cv/video/1920x1080.avi", "highgui/video/big_buck_bunny.avi", \ "highgui/video/big_buck_bunny.h264", "highgui/video/big_buck_bunny.h265", "highgui/video/big_buck_bunny.mpg" INSTANTIATE_TEST_CASE_P(CUDA_Codec, Video, testing::Combine( ALL_DEVICES, - testing::Values(VIDEO_SRC))); + testing::Values(VIDEO_SRC_R))); + +#define VIDEO_SRC_RW "highgui/video/big_buck_bunny.h264", "highgui/video/big_buck_bunny.h265" + +INSTANTIATE_TEST_CASE_P(CUDA_Codec, VideoReadWrite, testing::Combine( + ALL_DEVICES, + testing::Values(VIDEO_SRC_RW))); #endif // HAVE_NVCUVID || HAVE_NVCUVENC }} // namespace From f17c496240ef7ad965e5eaa487b89723515e84de Mon Sep 17 00:00:00 2001 From: cudawarped Date: Wed, 3 Nov 2021 11:20:45 +0000 Subject: [PATCH 3/9] Prevent adding parameter sets twice and add zero padding to output files to that they play in vlc. Notes: VideoCapture - returns mpeg as the codec for mpeg4 files, so files written as .m4v from mpeg4 sources cannot currently be decoded. This is also true for AV1 sources where cap.get(CAP_PROP_FOURCC) returns 0. Added mpeg4 test file which can be decoded when VideoCapture adds the extra_data. --- modules/cudacodec/src/ffmpeg_video_source.cpp | 60 ++++++++++++++++--- modules/cudacodec/test/test_video.cpp | 2 +- 2 files changed, 53 insertions(+), 9 deletions(-) diff --git a/modules/cudacodec/src/ffmpeg_video_source.cpp b/modules/cudacodec/src/ffmpeg_video_source.cpp index 1c381938f9..b31b52064e 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.cpp +++ b/modules/cudacodec/src/ffmpeg_video_source.cpp @@ -116,6 +116,7 @@ void FourccToChromaFormat(const int pixelFormat, ChromaFormat &chromaFormat, int static std::string CodecToFileExtension(const Codec codec){ switch (codec) { + case(Codec::MPEG4): return ".m4v"; case(Codec::H264): return ".h264"; case(Codec::HEVC): return ".h265"; case(Codec::VP8): return ".vp8"; @@ -182,6 +183,22 @@ void cv::cudacodec::detail::FFmpegVideoSource::writeToFile(const std::string _fi restartRtspFileWrite = true; } +int StartCodeLen(unsigned char* data, const int sz) { + if (sz >= 3 && data[0] == 0 && data[1] == 0 && data[2] == 1) + return 3; + else if (sz >=4 && data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1) + return 4; + else + return 0; +} + +bool ParamSetsExist(unsigned char* parameterSets, const int szParameterSets, unsigned char* data, const int szData) { + const int paramSetStartCodeLen = StartCodeLen(parameterSets, szParameterSets); + const int packetStartCodeLen = StartCodeLen(data, szData); + // weak test to see if the parameter set has already been included in the RTP stream + return paramSetStartCodeLen != 0 && packetStartCodeLen != 0 && parameterSets[paramSetStartCodeLen] == data[packetStartCodeLen]; +} + bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** data, size_t* size) { std::lock_guard lck(mtx); @@ -189,14 +206,28 @@ bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** dat *data = rawFrame.data; *size = rawFrame.total(); - // always grab param sets if available in case writeToFile is called after construction - const int paramSetLen = cap.get(CAP_PROP_LF_PARAM_SET_LEN); - if (paramSetLen) { - parameterSets = Mat(1, paramSetLen, CV_8UC1); - memcpy(parameterSets.data, *data, paramSetLen); + // always grab parameter sets if available to allow writeToFile to be called after construction + int paramSetsLen = cap.get(CAP_PROP_LRF_EXTRA_DATA_LEN); + if (paramSetsLen) { + // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in front of parameter sets transmitted in response to + // DESCRIPE RTSP message, required for playback in media players such as vlc. + int rtspParamSetZeroBytePadding = 0; + if (format_.codec == Codec::H264 || format_.codec == Codec::HEVC) { + if (StartCodeLen(*data,*size) == 3) + rtspParamSetZeroBytePadding = 1; + } + parameterSets = Mat::zeros(1, paramSetsLen + rtspParamSetZeroBytePadding, CV_8UC1); + memcpy(parameterSets.data + rtspParamSetZeroBytePadding, *data, paramSetsLen); + if ((format_.codec == Codec::H264 || format_.codec == Codec::HEVC) + && ParamSetsExist(parameterSets.data, paramSetsLen, &(*data)[paramSetsLen], *size)) { + *data = &rawFrame.data[paramSetsLen]; + *size = rawFrame.total() - paramSetsLen; + } } - if (restartRtspFileWrite && cap.get(CAP_PROP_LF_KEY_FRAME)) { + int rtpParamSetZeroBytePadding = 0; + if (restartRtspFileWrite && cap.get(CAP_PROP_LRF_HAS_KEY_FRAME)) { + restartRtspFileWrite = false; if (file.is_open()) file.close(); if (autoDetectExt) @@ -204,9 +235,17 @@ bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** dat file.open(fileName, std::ios::binary); if (!file.is_open()) return false; - restartRtspFileWrite = false; - if (!paramSetLen && !parameterSets.empty()) + if (!parameterSets.empty()){ writeParameterSets = true; + if ((format_.codec == Codec::H264 || format_.codec == Codec::HEVC) + && ParamSetsExist(parameterSets.data, parameterSets.total(), *data, *size)) { + writeParameterSets = false; + // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in the RTP stream in front of parameter sets, + // required for playback in media players such as vlc + if (StartCodeLen(*data, *size) == 3) + rtpParamSetZeroBytePadding = 1; + } + } } if (file.is_open()) { @@ -214,7 +253,12 @@ bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** dat writeParameterSets = false; file.write((char*)parameterSets.data, parameterSets.total()); } + else if (rtpParamSetZeroBytePadding) { + const char tmp = 0x00; + file.write(&tmp, 1); + } file.write((char*)*data, *size); + } return *size != 0; } diff --git a/modules/cudacodec/test/test_video.cpp b/modules/cudacodec/test/test_video.cpp index 88a0495740..b4a586bea9 100644 --- a/modules/cudacodec/test/test_video.cpp +++ b/modules/cudacodec/test/test_video.cpp @@ -171,7 +171,7 @@ CUDA_TEST_P(Video, Writer) #endif // _WIN32, HAVE_NVCUVENC -#define VIDEO_SRC_R "cv/video/768x576.avi", "cv/video/1920x1080.avi", "highgui/video/big_buck_bunny.avi", \ +#define VIDEO_SRC_R "highgui/video/big_buck_bunny.mp4", "cv/video/768x576.avi", "cv/video/1920x1080.avi", "highgui/video/big_buck_bunny.avi", \ "highgui/video/big_buck_bunny.h264", "highgui/video/big_buck_bunny.h265", "highgui/video/big_buck_bunny.mpg" INSTANTIATE_TEST_CASE_P(CUDA_Codec, Video, testing::Combine( ALL_DEVICES, From 2ee30508147d5874a1541c484b488c7b79c2815f Mon Sep 17 00:00:00 2001 From: cudawarped Date: Wed, 10 Nov 2021 17:33:33 +0000 Subject: [PATCH 4/9] Update to account for the extraData being passed from cap.retrieve instead of appended to the first packet. --- modules/cudacodec/src/ffmpeg_video_source.cpp | 68 ++++++++++--------- modules/cudacodec/src/ffmpeg_video_source.hpp | 4 +- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/modules/cudacodec/src/ffmpeg_video_source.cpp b/modules/cudacodec/src/ffmpeg_video_source.cpp index b31b52064e..271d1a6874 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.cpp +++ b/modules/cudacodec/src/ffmpeg_video_source.cpp @@ -205,28 +205,41 @@ bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** dat cap >> rawFrame; *data = rawFrame.data; *size = rawFrame.total(); - - // always grab parameter sets if available to allow writeToFile to be called after construction - int paramSetsLen = cap.get(CAP_PROP_LRF_EXTRA_DATA_LEN); - if (paramSetsLen) { - // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in front of parameter sets transmitted in response to - // DESCRIPE RTSP message, required for playback in media players such as vlc. - int rtspParamSetZeroBytePadding = 0; - if (format_.codec == Codec::H264 || format_.codec == Codec::HEVC) { - if (StartCodeLen(*data,*size) == 3) - rtspParamSetZeroBytePadding = 1; - } - parameterSets = Mat::zeros(1, paramSetsLen + rtspParamSetZeroBytePadding, CV_8UC1); - memcpy(parameterSets.data + rtspParamSetZeroBytePadding, *data, paramSetsLen); - if ((format_.codec == Codec::H264 || format_.codec == Codec::HEVC) - && ParamSetsExist(parameterSets.data, paramSetsLen, &(*data)[paramSetsLen], *size)) { - *data = &rawFrame.data[paramSetsLen]; - *size = rawFrame.total() - paramSetsLen; + int rtpParamSetZeroBytePadding = 0, rtspParamSetZeroBytePadding = 0; + bool writeParameterSets = false; + const bool startRtspFileWrite = restartRtspFileWrite && cap.get(CAP_PROP_LRF_HAS_KEY_FRAME); + if (iFrame++ == 0 || startRtspFileWrite) { + Mat tmpExtraData; + cap.retrieve(tmpExtraData, 1); + if (tmpExtraData.total()) { + if (format_.codec == Codec::H264 || format_.codec == Codec::HEVC) { + // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in front of parameter sets transmitted in response to + // DESCRIPE RTSP message, required for playback in media players such as vlc. + if (StartCodeLen(tmpExtraData.data, tmpExtraData.total()) == 3) + rtspParamSetZeroBytePadding = 1; + if (ParamSetsExist(tmpExtraData.data, tmpExtraData.total(), *data, *size)) { + // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in the RTP stream in front of parameter sets, + // required for playback in media players such as vlc. + if (StartCodeLen(*data, *size) == 3) + rtpParamSetZeroBytePadding = 1; + } + else { + parameterSets = tmpExtraData.clone(); + writeParameterSets = true; + } + } + else if (format_.codec == Codec::MPEG4) { + const size_t newSz = tmpExtraData.total() + *size - 3; + dataWithHeader = Mat(1, newSz, CV_8UC1); + memcpy(dataWithHeader.data, tmpExtraData.data, tmpExtraData.total()); + memcpy(dataWithHeader.data + tmpExtraData.total(), (*data) + 3, *size - 3); + *data = dataWithHeader.data; + *size = newSz; + } } } - int rtpParamSetZeroBytePadding = 0; - if (restartRtspFileWrite && cap.get(CAP_PROP_LRF_HAS_KEY_FRAME)) { + if (startRtspFileWrite) { restartRtspFileWrite = false; if (file.is_open()) file.close(); @@ -235,22 +248,15 @@ bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** dat file.open(fileName, std::ios::binary); if (!file.is_open()) return false; - if (!parameterSets.empty()){ - writeParameterSets = true; - if ((format_.codec == Codec::H264 || format_.codec == Codec::HEVC) - && ParamSetsExist(parameterSets.data, parameterSets.total(), *data, *size)) { - writeParameterSets = false; - // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in the RTP stream in front of parameter sets, - // required for playback in media players such as vlc - if (StartCodeLen(*data, *size) == 3) - rtpParamSetZeroBytePadding = 1; - } - } } if (file.is_open()) { if (writeParameterSets) { writeParameterSets = false; + if (rtspParamSetZeroBytePadding) { + const char tmp = 0x00; + file.write(&tmp, 1); + } file.write((char*)parameterSets.data, parameterSets.total()); } else if (rtpParamSetZeroBytePadding) { @@ -258,8 +264,8 @@ bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** dat file.write(&tmp, 1); } file.write((char*)*data, *size); - } + return *size != 0; } diff --git a/modules/cudacodec/src/ffmpeg_video_source.hpp b/modules/cudacodec/src/ffmpeg_video_source.hpp index 7401baeca1..194df7b561 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.hpp +++ b/modules/cudacodec/src/ffmpeg_video_source.hpp @@ -67,12 +67,12 @@ class FFmpegVideoSource : public RawVideoSource private: FormatInfo format_; VideoCapture cap; - Mat rawFrame, parameterSets; + Mat rawFrame, parameterSets, dataWithHeader; std::string fileName; std::ofstream file; bool autoDetectExt = false; bool restartRtspFileWrite = false; - bool writeParameterSets = false; + int iFrame = 0; std::mutex mtx; }; From f718d8c1f213f9ebf3c8a5024a3af5852eca5914 Mon Sep 17 00:00:00 2001 From: cudawarped Date: Wed, 17 Nov 2021 17:07:24 +0000 Subject: [PATCH 5/9] Update to be compatible with changes to VideoCapture --- modules/cudacodec/src/ffmpeg_video_source.cpp | 49 ++++++++++--------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/modules/cudacodec/src/ffmpeg_video_source.cpp b/modules/cudacodec/src/ffmpeg_video_source.cpp index 271d1a6874..c0393f4fb7 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.cpp +++ b/modules/cudacodec/src/ffmpeg_video_source.cpp @@ -210,32 +210,35 @@ bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** dat const bool startRtspFileWrite = restartRtspFileWrite && cap.get(CAP_PROP_LRF_HAS_KEY_FRAME); if (iFrame++ == 0 || startRtspFileWrite) { Mat tmpExtraData; - cap.retrieve(tmpExtraData, 1); - if (tmpExtraData.total()) { - if (format_.codec == Codec::H264 || format_.codec == Codec::HEVC) { - // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in front of parameter sets transmitted in response to - // DESCRIPE RTSP message, required for playback in media players such as vlc. - if (StartCodeLen(tmpExtraData.data, tmpExtraData.total()) == 3) - rtspParamSetZeroBytePadding = 1; - if (ParamSetsExist(tmpExtraData.data, tmpExtraData.total(), *data, *size)) { - // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in the RTP stream in front of parameter sets, - // required for playback in media players such as vlc. - if (StartCodeLen(*data, *size) == 3) - rtpParamSetZeroBytePadding = 1; + const int codecExtradataIndex = (int)cap.get(CAP_PROP_CODEC_EXTRADATA_INDEX); + if (codecExtradataIndex) { + cap.retrieve(tmpExtraData, codecExtradataIndex); + if (tmpExtraData.total()) { + if (format_.codec == Codec::H264 || format_.codec == Codec::HEVC) { + // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in front of parameter sets transmitted in response to + // DESCRIPE RTSP message, required for playback in media players such as vlc. + if (StartCodeLen(tmpExtraData.data, tmpExtraData.total()) == 3) + rtspParamSetZeroBytePadding = 1; + if (ParamSetsExist(tmpExtraData.data, tmpExtraData.total(), *data, *size)) { + // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in the RTP stream in front of parameter sets, + // required for playback in media players such as vlc. + if (StartCodeLen(*data, *size) == 3) + rtpParamSetZeroBytePadding = 1; + } + else { + parameterSets = tmpExtraData.clone(); + writeParameterSets = true; + } } - else { - parameterSets = tmpExtraData.clone(); - writeParameterSets = true; + else if (format_.codec == Codec::MPEG4) { + const size_t newSz = tmpExtraData.total() + *size - 3; + dataWithHeader = Mat(1, newSz, CV_8UC1); + memcpy(dataWithHeader.data, tmpExtraData.data, tmpExtraData.total()); + memcpy(dataWithHeader.data + tmpExtraData.total(), (*data) + 3, *size - 3); + *data = dataWithHeader.data; + *size = newSz; } } - else if (format_.codec == Codec::MPEG4) { - const size_t newSz = tmpExtraData.total() + *size - 3; - dataWithHeader = Mat(1, newSz, CV_8UC1); - memcpy(dataWithHeader.data, tmpExtraData.data, tmpExtraData.total()); - memcpy(dataWithHeader.data + tmpExtraData.total(), (*data) + 3, *size - 3); - *data = dataWithHeader.data; - *size = newSz; - } } } From 7ae7eb9a853013cc91b0bc38c6b161da64482c17 Mon Sep 17 00:00:00 2001 From: cudawarped Date: Wed, 17 Nov 2021 17:15:53 +0000 Subject: [PATCH 6/9] Remove redundant test. --- modules/cudacodec/src/ffmpeg_video_source.cpp | 48 +++++++++---------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/modules/cudacodec/src/ffmpeg_video_source.cpp b/modules/cudacodec/src/ffmpeg_video_source.cpp index c0393f4fb7..edd3d83db4 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.cpp +++ b/modules/cudacodec/src/ffmpeg_video_source.cpp @@ -211,34 +211,32 @@ bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** dat if (iFrame++ == 0 || startRtspFileWrite) { Mat tmpExtraData; const int codecExtradataIndex = (int)cap.get(CAP_PROP_CODEC_EXTRADATA_INDEX); - if (codecExtradataIndex) { - cap.retrieve(tmpExtraData, codecExtradataIndex); - if (tmpExtraData.total()) { - if (format_.codec == Codec::H264 || format_.codec == Codec::HEVC) { - // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in front of parameter sets transmitted in response to - // DESCRIPE RTSP message, required for playback in media players such as vlc. - if (StartCodeLen(tmpExtraData.data, tmpExtraData.total()) == 3) - rtspParamSetZeroBytePadding = 1; - if (ParamSetsExist(tmpExtraData.data, tmpExtraData.total(), *data, *size)) { - // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in the RTP stream in front of parameter sets, - // required for playback in media players such as vlc. - if (StartCodeLen(*data, *size) == 3) - rtpParamSetZeroBytePadding = 1; - } - else { - parameterSets = tmpExtraData.clone(); - writeParameterSets = true; - } + cap.retrieve(tmpExtraData, codecExtradataIndex); + if (tmpExtraData.total()) { + if (format_.codec == Codec::H264 || format_.codec == Codec::HEVC) { + // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in front of parameter sets transmitted in response to + // DESCRIPE RTSP message, required for playback in media players such as vlc. + if (StartCodeLen(tmpExtraData.data, tmpExtraData.total()) == 3) + rtspParamSetZeroBytePadding = 1; + if (ParamSetsExist(tmpExtraData.data, tmpExtraData.total(), *data, *size)) { + // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in the RTP stream in front of parameter sets, + // required for playback in media players such as vlc. + if (StartCodeLen(*data, *size) == 3) + rtpParamSetZeroBytePadding = 1; } - else if (format_.codec == Codec::MPEG4) { - const size_t newSz = tmpExtraData.total() + *size - 3; - dataWithHeader = Mat(1, newSz, CV_8UC1); - memcpy(dataWithHeader.data, tmpExtraData.data, tmpExtraData.total()); - memcpy(dataWithHeader.data + tmpExtraData.total(), (*data) + 3, *size - 3); - *data = dataWithHeader.data; - *size = newSz; + else { + parameterSets = tmpExtraData.clone(); + writeParameterSets = true; } } + else if (format_.codec == Codec::MPEG4) { + const size_t newSz = tmpExtraData.total() + *size - 3; + dataWithHeader = Mat(1, newSz, CV_8UC1); + memcpy(dataWithHeader.data, tmpExtraData.data, tmpExtraData.total()); + memcpy(dataWithHeader.data + tmpExtraData.total(), (*data) + 3, *size - 3); + *data = dataWithHeader.data; + *size = newSz; + } } } From a179fe441c22027d012d7f99fad29eacd7ad1a59 Mon Sep 17 00:00:00 2001 From: cudawarped Date: Thu, 25 Nov 2021 16:56:56 +0000 Subject: [PATCH 7/9] Add check to ensure retrieve is successful. --- modules/cudacodec/src/ffmpeg_video_source.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/cudacodec/src/ffmpeg_video_source.cpp b/modules/cudacodec/src/ffmpeg_video_source.cpp index edd3d83db4..d0db602a16 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.cpp +++ b/modules/cudacodec/src/ffmpeg_video_source.cpp @@ -205,14 +205,14 @@ bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** dat cap >> rawFrame; *data = rawFrame.data; *size = rawFrame.total(); + int rtpParamSetZeroBytePadding = 0, rtspParamSetZeroBytePadding = 0; bool writeParameterSets = false; const bool startRtspFileWrite = restartRtspFileWrite && cap.get(CAP_PROP_LRF_HAS_KEY_FRAME); if (iFrame++ == 0 || startRtspFileWrite) { Mat tmpExtraData; const int codecExtradataIndex = (int)cap.get(CAP_PROP_CODEC_EXTRADATA_INDEX); - cap.retrieve(tmpExtraData, codecExtradataIndex); - if (tmpExtraData.total()) { + if (cap.retrieve(tmpExtraData, codecExtradataIndex) && tmpExtraData.total()) { if (format_.codec == Codec::H264 || format_.codec == Codec::HEVC) { // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in front of parameter sets transmitted in response to // DESCRIPE RTSP message, required for playback in media players such as vlc. From 65a09c3bca539688d456ab81b9d074066ee47635 Mon Sep 17 00:00:00 2001 From: cudawarped Date: Wed, 8 Dec 2021 16:33:01 +0000 Subject: [PATCH 8/9] Remove writeToFile and allow VideoReader to return raw encoded data at the same time as decoded frames. --- .../cudacodec/include/opencv2/cudacodec.hpp | 89 +++++++---- modules/cudacodec/src/cuvid_video_source.cpp | 2 +- modules/cudacodec/src/cuvid_video_source.hpp | 1 - modules/cudacodec/src/ffmpeg_video_source.cpp | 137 +++++------------ modules/cudacodec/src/ffmpeg_video_source.hpp | 15 +- modules/cudacodec/src/frame_queue.cpp | 17 +- modules/cudacodec/src/frame_queue.hpp | 16 +- modules/cudacodec/src/video_decoder.hpp | 1 - modules/cudacodec/src/video_parser.cpp | 9 +- modules/cudacodec/src/video_parser.hpp | 3 +- modules/cudacodec/src/video_reader.cpp | 112 ++++++++++++-- modules/cudacodec/src/video_source.cpp | 28 ++-- modules/cudacodec/src/video_source.hpp | 34 ++-- modules/cudacodec/test/test_video.cpp | 145 ++++++++++++++++-- 14 files changed, 404 insertions(+), 205 deletions(-) diff --git a/modules/cudacodec/include/opencv2/cudacodec.hpp b/modules/cudacodec/include/opencv2/cudacodec.hpp index f98fa817d9..61dec07961 100644 --- a/modules/cudacodec/include/opencv2/cudacodec.hpp +++ b/modules/cudacodec/include/opencv2/cudacodec.hpp @@ -308,6 +308,17 @@ struct FormatInfo DeinterlaceMode deinterlaceMode; }; +/** @brief cv::cudacodec::VideoReader generic properties identifier. +*/ +enum class VideoReaderProps { + PROP_DECODED_FRAME_IDX = 0, //!< Index for retrieving the decoded frame using retrieve(). + PROP_EXTRA_DATA_INDEX = 1, //!< Index for retrieving the extra data associated with a video source using retrieve(). + PROP_RAW_PACKAGES_BASE_INDEX = 2, //!< Base index for retrieving raw encoded data using retrieve(). + PROP_NUMBER_OF_RAW_PACKAGES_SINCE_LAST_GRAB = 3, //!< Number of raw packages recieved since the last call to grab(). + PROP_RAW_MODE = 4, //!< Status of raw mode. + PROP_LRF_HAS_KEY_FRAME = 5 //!< FFmpeg source only - Indicates whether the Last Raw Frame (LRF), output from VideoReader::retrieve() when VideoReader is initialized in raw mode, contains encoded data for a key frame. +}; + /** @brief Video reader interface. @note @@ -330,22 +341,47 @@ class CV_EXPORTS_W VideoReader */ virtual FormatInfo format() const = 0; - /** @brief Signals VideoReader to start writing the raw bitstream to the given file from the next key frame. + /** @brief Grabs the next frame from the video source. + + @return `true` (non-zero) in the case of success. + + The method/function grabs the next frame from video file or camera and returns true (non-zero) in + the case of success. + + The primary use of the function is for reading both the encoded and decoded video data when rawMode is enabled. With rawMode enabled + retrieve() can be called following grab() to retrieve all the data associated with the current video source since the last call to grab() or the creation of the VideoReader. + */ + CV_WRAP virtual bool grab(Stream& stream = Stream::Null()) = 0; + + /** @brief Returns previously grabbed video data. + + @param [out] image The returned data which depends on the provided idx. If there is no new data since the last call to grab() the image will be empty. + @param idx Determins the returned data inside image. The returned data can be the: + Decoded frame, idx = get(PROP_DECODED_FRAME_IDX). + Extra data if available, idx = get(PROP_EXTRA_DATA_INDEX). + Raw encoded data package. To retrieve package i, idx = get(PROP_RAW_PACKAGES_BASE_INDEX) + i with i < get(PROP_NUMBER_OF_RAW_PACKAGES_SINCE_LAST_GRAB) + @return `false` if no frames has been grabbed - @param filename Full path to the output file. Each call to this with a new filename, filenameNew will wait for - the next key frame, stop writing to filename and start writing to filenameNew. If VideoReader is already writing - to a file and filenameNew is empty, writing to filename will stop imidiately. - @param autoDetectExt Automatically detect and append the codec extension to filename before writing. + The method returns data associated with the current video source since the last call to grab() or the creation of the VideoReader. If no data is present + the method returns false and the function returns an empty image. + */ + CV_WRAP virtual bool retrieve(CV_OUT OutputArray frame, const size_t idx = static_cast(VideoReaderProps::PROP_DECODED_FRAME_IDX)) const = 0; - The function is intended to be used when streaming from an RTSP source where it is either - a) desirable to store the original streamed data, or - b) the overhead of re-encoding the decoded frame is undesirable. + /** @brief Sets a property in the VideoReader. - @note This should only be used when streaming from rtsp and is not guaranteed to work when reading from a - file, especially from container formats avi, mp4 etc. If the filename provided is invalid, cannot be opened - or written to, the first call to nextFrame() after calling this function will return false. + @param property Property identifier from cv::cudacodec::VideoReaderProps (eg. cv::cudacodec::PROP_DECODED_FRAME_IDX, cv::cudacodec::PROP_EXTRA_DATA_INDEX, ...) + @param propertyVal Value of the property. + @return `true` if the property has been set. */ - CV_WRAP virtual void writeToFile(const std::string filename, const bool autoDetectExt = false) = 0; + CV_WRAP virtual bool set(const VideoReaderProps property, const double propertyVal) = 0; + + /** @brief Returns the specified VideoReader property + + @param property Property identifier from cv::cudacodec::VideoReaderProps (eg. cv::cudacodec::PROP_DECODED_FRAME_IDX, cv::cudacodec::PROP_EXTRA_DATA_INDEX, ...) + @param propertyVal Optional value for the property. + @return Value for the specified property. Value -1 is returned when querying a property that is not supported. + */ + CV_WRAP virtual int get(const VideoReaderProps property, const int propertyVal = -1) const = 0; }; /** @brief Interface for video demultiplexing. : @@ -364,6 +400,10 @@ class CV_EXPORTS_W RawVideoSource */ virtual bool getNextPacket(unsigned char** data, size_t* size) = 0; + /** @brief Returns true if the last packet contained a key frame. + */ + virtual bool lastPacketContainsKeyFrame() const { return false; } + /** @brief Returns information about video file format. */ virtual FormatInfo format() const = 0; @@ -372,37 +412,26 @@ class CV_EXPORTS_W RawVideoSource */ virtual void updateFormat(const FormatInfo& videoFormat) = 0; - /** @brief Signals RawVideoSource to start writing the raw bitstream to the given file from the next key frame. - - @param filename Full path to the output file. Each call to this with a new filename, filenameNew will wait for - the next key frame, stop writing to filename and start writing to filenameNew. If VideoReader is already writing - to a file and filenameNew is empty, writing to filename will stop imidiately. - @param autoDetectExt Automatically detect and append the codec extension to filename before writing. + /** @brief Returns any extra data associated with the video source. - The function is intended to be used when streaming from an RTSP source where it is either - a) desirable to store the original streamed data, or - b) the overhead of re-encoding the decoded frame is undesirable. - - @note This should only be used when streaming from rtsp and is not guaranteed to work when reading from a - file, especially from container formats avi, mp4 etc. If the filename provided is invalid, cannot be opened - or written to, the first call to nextFrame() after calling this function will return false. + @param extraData 1D cv::Mat containing the extra data if it exists. */ - virtual void writeToFile(const std::string filename, const bool autoDetectExt = false) = 0; + virtual void getExtraData(cv::Mat& extraData) const = 0; }; /** @brief Creates video reader. @param filename Name of the input video file. -@param filenameToWrite Full path to the raw output file. If empty the raw encoded video data will not be written. -@param autoDetectExt Automatically detect and append the codec extension to filename. +@param rawMode Allow the raw encoded data which has been read up until the last call to grab() to be retrieved by calling retrieve(rawData,RAW_DATA_IDX). FFMPEG is used to read videos. User can implement own demultiplexing with cudacodec::RawVideoSource */ -CV_EXPORTS_W Ptr createVideoReader(const String& filename, const String filenameToWrite = "", const bool autoDetectExt = false); +CV_EXPORTS_W Ptr createVideoReader(const String& filename, const bool rawMode = false); + /** @overload @param source RAW video source implemented by user. */ -CV_EXPORTS_W Ptr createVideoReader(const Ptr& source); +CV_EXPORTS_W Ptr createVideoReader(const Ptr& source, const bool rawMode = false); //! @} diff --git a/modules/cudacodec/src/cuvid_video_source.cpp b/modules/cudacodec/src/cuvid_video_source.cpp index 72b72f37dd..3d7bbbb8c0 100644 --- a/modules/cudacodec/src/cuvid_video_source.cpp +++ b/modules/cudacodec/src/cuvid_video_source.cpp @@ -120,7 +120,7 @@ int CUDAAPI cv::cudacodec::detail::CuvidVideoSource::HandleVideoData(void* userD { CuvidVideoSource* thiz = static_cast(userData); - return thiz->parseVideoData(packet->payload, packet->payload_size, (packet->flags & CUVID_PKT_ENDOFSTREAM) != 0); + return thiz->parseVideoData(packet->payload, packet->payload_size, thiz->RawModeEnabled(), false, (packet->flags & CUVID_PKT_ENDOFSTREAM) != 0); } #endif // HAVE_NVCUVID diff --git a/modules/cudacodec/src/cuvid_video_source.hpp b/modules/cudacodec/src/cuvid_video_source.hpp index fddea2d7a3..21b28980a2 100644 --- a/modules/cudacodec/src/cuvid_video_source.hpp +++ b/modules/cudacodec/src/cuvid_video_source.hpp @@ -56,7 +56,6 @@ class CuvidVideoSource : public VideoSource FormatInfo format() const CV_OVERRIDE; void updateFormat(const FormatInfo& videoFormat) CV_OVERRIDE; - void writeToFile(const std::string filename, const bool autoDetectExt = false) CV_OVERRIDE {}; void start() CV_OVERRIDE; void stop() CV_OVERRIDE; bool isStarted() const CV_OVERRIDE; diff --git a/modules/cudacodec/src/ffmpeg_video_source.cpp b/modules/cudacodec/src/ffmpeg_video_source.cpp index d0db602a16..4efbe775a0 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.cpp +++ b/modules/cudacodec/src/ffmpeg_video_source.cpp @@ -113,19 +113,23 @@ void FourccToChromaFormat(const int pixelFormat, ChromaFormat &chromaFormat, int } } -static -std::string CodecToFileExtension(const Codec codec){ - switch (codec) { - case(Codec::MPEG4): return ".m4v"; - case(Codec::H264): return ".h264"; - case(Codec::HEVC): return ".h265"; - case(Codec::VP8): return ".vp8"; - case(Codec::VP9): return ".vp9"; - default: return ""; - } +int StartCodeLen(unsigned char* data, const int sz) { + if (sz >= 3 && data[0] == 0 && data[1] == 0 && data[2] == 1) + return 3; + else if (sz >= 4 && data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1) + return 4; + else + return 0; +} + +bool ParamSetsExist(unsigned char* parameterSets, const int szParameterSets, unsigned char* data, const int szData) { + const int paramSetStartCodeLen = StartCodeLen(parameterSets, szParameterSets); + const int packetStartCodeLen = StartCodeLen(data, szData); + // weak test to see if the parameter set has already been included in the RTP stream + return paramSetStartCodeLen != 0 && packetStartCodeLen != 0 && parameterSets[paramSetStartCodeLen] == data[packetStartCodeLen]; } -cv::cudacodec::detail::FFmpegVideoSource::FFmpegVideoSource(const String& fname, const String& filenameToWrite, const bool autoDetectExt) +cv::cudacodec::detail::FFmpegVideoSource::FFmpegVideoSource(const String& fname) { if (!videoio_registry::hasBackend(CAP_FFMPEG)) CV_Error(Error::StsNotImplemented, "FFmpeg backend not found"); @@ -138,8 +142,10 @@ cv::cudacodec::detail::FFmpegVideoSource::FFmpegVideoSource(const String& fname, CV_Error(Error::StsUnsupportedFormat, "Fetching of RAW video streams is not supported"); CV_Assert(cap.get(CAP_PROP_FORMAT) == -1); - if (!filenameToWrite.empty()) - writeToFile(filenameToWrite, autoDetectExt); + const int codecExtradataIndex = static_cast(cap.get(CAP_PROP_CODEC_EXTRADATA_INDEX)); + Mat tmpExtraData; + if (cap.retrieve(tmpExtraData, codecExtradataIndex) && tmpExtraData.total()) + extraData = tmpExtraData.clone(); int codec = (int)cap.get(CAP_PROP_FOURCC); int pixelFormat = (int)cap.get(CAP_PROP_CODEC_PIXEL_FORMAT); @@ -170,104 +176,31 @@ void cv::cudacodec::detail::FFmpegVideoSource::updateFormat(const FormatInfo& vi format_.valid = true; } -void cv::cudacodec::detail::FFmpegVideoSource::writeToFile(const std::string _filename, const bool _autoDetectExt) { - std::lock_guard lck(mtx); - fileName = _filename; - if(fileName.empty()){ - if (file.is_open()) - file.close(); - restartRtspFileWrite = false; - return; - } - autoDetectExt = _autoDetectExt; - restartRtspFileWrite = true; -} - -int StartCodeLen(unsigned char* data, const int sz) { - if (sz >= 3 && data[0] == 0 && data[1] == 0 && data[2] == 1) - return 3; - else if (sz >=4 && data[0] == 0 && data[1] == 0 && data[2] == 0 && data[3] == 1) - return 4; - else - return 0; -} - -bool ParamSetsExist(unsigned char* parameterSets, const int szParameterSets, unsigned char* data, const int szData) { - const int paramSetStartCodeLen = StartCodeLen(parameterSets, szParameterSets); - const int packetStartCodeLen = StartCodeLen(data, szData); - // weak test to see if the parameter set has already been included in the RTP stream - return paramSetStartCodeLen != 0 && packetStartCodeLen != 0 && parameterSets[paramSetStartCodeLen] == data[packetStartCodeLen]; -} - bool cv::cudacodec::detail::FFmpegVideoSource::getNextPacket(unsigned char** data, size_t* size) { - std::lock_guard lck(mtx); cap >> rawFrame; *data = rawFrame.data; *size = rawFrame.total(); - - int rtpParamSetZeroBytePadding = 0, rtspParamSetZeroBytePadding = 0; - bool writeParameterSets = false; - const bool startRtspFileWrite = restartRtspFileWrite && cap.get(CAP_PROP_LRF_HAS_KEY_FRAME); - if (iFrame++ == 0 || startRtspFileWrite) { - Mat tmpExtraData; - const int codecExtradataIndex = (int)cap.get(CAP_PROP_CODEC_EXTRADATA_INDEX); - if (cap.retrieve(tmpExtraData, codecExtradataIndex) && tmpExtraData.total()) { - if (format_.codec == Codec::H264 || format_.codec == Codec::HEVC) { - // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in front of parameter sets transmitted in response to - // DESCRIPE RTSP message, required for playback in media players such as vlc. - if (StartCodeLen(tmpExtraData.data, tmpExtraData.total()) == 3) - rtspParamSetZeroBytePadding = 1; - if (ParamSetsExist(tmpExtraData.data, tmpExtraData.total(), *data, *size)) { - // ensure zero_byte (Annex B of the ITU-T H.264[5]) is present in the RTP stream in front of parameter sets, - // required for playback in media players such as vlc. - if (StartCodeLen(*data, *size) == 3) - rtpParamSetZeroBytePadding = 1; - } - else { - parameterSets = tmpExtraData.clone(); - writeParameterSets = true; - } - } - else if (format_.codec == Codec::MPEG4) { - const size_t newSz = tmpExtraData.total() + *size - 3; - dataWithHeader = Mat(1, newSz, CV_8UC1); - memcpy(dataWithHeader.data, tmpExtraData.data, tmpExtraData.total()); - memcpy(dataWithHeader.data + tmpExtraData.total(), (*data) + 3, *size - 3); - *data = dataWithHeader.data; - *size = newSz; - } - } - } - - if (startRtspFileWrite) { - restartRtspFileWrite = false; - if (file.is_open()) - file.close(); - if (autoDetectExt) - fileName += CodecToFileExtension(format_.codec); - file.open(fileName, std::ios::binary); - if (!file.is_open()) - return false; - } - - if (file.is_open()) { - if (writeParameterSets) { - writeParameterSets = false; - if (rtspParamSetZeroBytePadding) { - const char tmp = 0x00; - file.write(&tmp, 1); - } - file.write((char*)parameterSets.data, parameterSets.total()); - } - else if (rtpParamSetZeroBytePadding) { - const char tmp = 0x00; - file.write(&tmp, 1); + if (iFrame++ == 0 && extraData.total()) { + if (format_.codec == Codec::MPEG4 || + ((format_.codec == Codec::H264 || format_.codec == Codec::HEVC) && !ParamSetsExist(extraData.data, extraData.total(), *data, *size))) + { + const size_t nBytesToTrimFromData = format_.codec == Codec::MPEG4 ? 3 : 0; + const size_t newSz = extraData.total() + *size - nBytesToTrimFromData; + dataWithHeader = Mat(1, newSz, CV_8UC1); + memcpy(dataWithHeader.data, extraData.data, extraData.total()); + memcpy(dataWithHeader.data + extraData.total(), (*data) + nBytesToTrimFromData, *size - nBytesToTrimFromData); + *data = dataWithHeader.data; + *size = newSz; } - file.write((char*)*data, *size); } return *size != 0; } +bool cv::cudacodec::detail::FFmpegVideoSource::lastPacketContainsKeyFrame() const +{ + return cap.get(CAP_PROP_LRF_HAS_KEY_FRAME); +} + #endif // HAVE_CUDA diff --git a/modules/cudacodec/src/ffmpeg_video_source.hpp b/modules/cudacodec/src/ffmpeg_video_source.hpp index 194df7b561..e8346331e4 100644 --- a/modules/cudacodec/src/ffmpeg_video_source.hpp +++ b/modules/cudacodec/src/ffmpeg_video_source.hpp @@ -44,8 +44,6 @@ #ifndef __FFMPEG_VIDEO_SOURCE_HPP__ #define __FFMPEG_VIDEO_SOURCE_HPP__ -#include - #include "opencv2/cudacodec.hpp" namespace cv { namespace cudacodec { namespace detail { @@ -53,27 +51,24 @@ namespace cv { namespace cudacodec { namespace detail { class FFmpegVideoSource : public RawVideoSource { public: - FFmpegVideoSource(const String& fname, const String& filenameToWrite, const bool autoDetectExt = false); + FFmpegVideoSource(const String& fname); ~FFmpegVideoSource(); bool getNextPacket(unsigned char** data, size_t* size) CV_OVERRIDE; + bool lastPacketContainsKeyFrame() const; + FormatInfo format() const CV_OVERRIDE; void updateFormat(const FormatInfo& videoFormat) CV_OVERRIDE; - void writeToFile(const std::string filename, const bool autoDetectExt = false) CV_OVERRIDE; + void getExtraData(cv::Mat& _extraData) const CV_OVERRIDE { _extraData = extraData; } private: FormatInfo format_; VideoCapture cap; - Mat rawFrame, parameterSets, dataWithHeader; - std::string fileName; - std::ofstream file; - bool autoDetectExt = false; - bool restartRtspFileWrite = false; + Mat rawFrame, extraData, dataWithHeader; int iFrame = 0; - std::mutex mtx; }; }}} diff --git a/modules/cudacodec/src/frame_queue.cpp b/modules/cudacodec/src/frame_queue.cpp index 87fb1e7216..f5d8eb9191 100644 --- a/modules/cudacodec/src/frame_queue.cpp +++ b/modules/cudacodec/src/frame_queue.cpp @@ -45,6 +45,11 @@ #ifdef HAVE_NVCUVID +RawPacket::RawPacket(const unsigned char* _data, const size_t _size, const bool _containsKeyFrame) : size(_size), containsKeyFrame(_containsKeyFrame) { + data = cv::makePtr(new unsigned char[size]); + memcpy(*data, _data, size); +}; + cv::cudacodec::detail::FrameQueue::~FrameQueue() { if (isFrameInUse_) delete[] isFrameInUse_; @@ -72,7 +77,7 @@ bool cv::cudacodec::detail::FrameQueue::waitUntilFrameAvailable(int pictureIndex return true; } -void cv::cudacodec::detail::FrameQueue::enqueue(const CUVIDPARSERDISPINFO* picParams) +void cv::cudacodec::detail::FrameQueue::enqueue(const CUVIDPARSERDISPINFO* picParams, const std::vector rawPackets) { // Mark the frame as 'in-use' so we don't re-use it for decoding until it is no longer needed // for display @@ -88,8 +93,10 @@ void cv::cudacodec::detail::FrameQueue::enqueue(const CUVIDPARSERDISPINFO* picPa if (framesInQueue_ < maxSz) { - int writePosition = (readPosition_ + framesInQueue_) % maxSz; + const int writePosition = (readPosition_ + framesInQueue_) % maxSz; displayQueue_.at(writePosition) = *picParams; + for (const auto& rawPacket : rawPackets) + rawPacketQueue.push(rawPacket); framesInQueue_++; isFramePlaced = true; } @@ -103,7 +110,7 @@ void cv::cudacodec::detail::FrameQueue::enqueue(const CUVIDPARSERDISPINFO* picPa } while (!isEndOfDecode()); } -bool cv::cudacodec::detail::FrameQueue::dequeue(CUVIDPARSERDISPINFO& displayInfo) +bool cv::cudacodec::detail::FrameQueue::dequeue(CUVIDPARSERDISPINFO& displayInfo, std::vector& rawPackets) { AutoLock autoLock(mtx_); @@ -111,6 +118,10 @@ bool cv::cudacodec::detail::FrameQueue::dequeue(CUVIDPARSERDISPINFO& displayInfo { int entry = readPosition_; displayInfo = displayQueue_.at(entry); + while (!rawPacketQueue.empty()) { + rawPackets.push_back(rawPacketQueue.front()); + rawPacketQueue.pop(); + } readPosition_ = (entry + 1) % maxSz; framesInQueue_--; return true; diff --git a/modules/cudacodec/src/frame_queue.hpp b/modules/cudacodec/src/frame_queue.hpp index 595c4fc17d..f5a9b34373 100644 --- a/modules/cudacodec/src/frame_queue.hpp +++ b/modules/cudacodec/src/frame_queue.hpp @@ -43,9 +43,20 @@ #ifndef __FRAME_QUEUE_HPP__ #define __FRAME_QUEUE_HPP__ +#include #include "opencv2/core/utility.hpp" +class RawPacket { +public: + RawPacket(const unsigned char* _data, const size_t _size = 0, const bool _containsKeyFrame = false); + unsigned char* Data() const { return *data; } + size_t size; + bool containsKeyFrame; +private: + cv::Ptr data = 0; +}; + namespace cv { namespace cudacodec { namespace detail { class FrameQueue @@ -63,7 +74,7 @@ class FrameQueue // available, the method returns false. bool waitUntilFrameAvailable(int pictureIndex); - void enqueue(const CUVIDPARSERDISPINFO* picParams); + void enqueue(const CUVIDPARSERDISPINFO* picParams, const std::vector rawPackets); // Deque the next frame. // Parameters: @@ -71,7 +82,7 @@ class FrameQueue // Returns: // true, if a new frame was returned, // false, if the queue was empty and no new frame could be returned. - bool dequeue(CUVIDPARSERDISPINFO& displayInfo); + bool dequeue(CUVIDPARSERDISPINFO& displayInfo, std::vector& rawPackets); void releaseFrame(const CUVIDPARSERDISPINFO& picParams) { isFrameInUse_[picParams.picture_index] = false; } @@ -85,6 +96,7 @@ class FrameQueue int readPosition_ = 0; std::vector< CUVIDPARSERDISPINFO> displayQueue_; int maxSz = 0; + std::queue rawPacketQueue; }; }}} diff --git a/modules/cudacodec/src/video_decoder.hpp b/modules/cudacodec/src/video_decoder.hpp index 452d16171e..3f59ed0b19 100644 --- a/modules/cudacodec/src/video_decoder.hpp +++ b/modules/cudacodec/src/video_decoder.hpp @@ -98,7 +98,6 @@ class VideoDecoder } private: - //Codec codec_ = NumCodecs; CUcontext ctx_ = 0; CUvideoctxlock lock_; CUvideodecoder decoder_ = 0; diff --git a/modules/cudacodec/src/video_parser.cpp b/modules/cudacodec/src/video_parser.cpp index 907ea2d7e7..7ed656625b 100644 --- a/modules/cudacodec/src/video_parser.cpp +++ b/modules/cudacodec/src/video_parser.cpp @@ -62,7 +62,7 @@ cv::cudacodec::detail::VideoParser::VideoParser(VideoDecoder* videoDecoder, Fram cuSafeCall( cuvidCreateVideoParser(&parser_, ¶ms) ); } -bool cv::cudacodec::detail::VideoParser::parseVideoData(const unsigned char* data, size_t size, bool endOfStream) +bool cv::cudacodec::detail::VideoParser::parseVideoData(const unsigned char* data, size_t size, const bool rawMode, const bool containsKeyFrame, bool endOfStream) { CUVIDSOURCEDATAPACKET packet; std::memset(&packet, 0, sizeof(CUVIDSOURCEDATAPACKET)); @@ -73,6 +73,9 @@ bool cv::cudacodec::detail::VideoParser::parseVideoData(const unsigned char* dat packet.payload_size = static_cast(size); packet.payload = data; + if (rawMode) + currentFramePackets.push_back(RawPacket(data, size, containsKeyFrame)); + if (cuvidParseVideoData(parser_, &packet) != CUDA_SUCCESS) { hasError_ = true; @@ -180,8 +183,8 @@ int CUDAAPI cv::cudacodec::detail::VideoParser::HandlePictureDisplay(void* userD thiz->unparsedPackets_ = 0; - thiz->frameQueue_->enqueue(picParams); - + thiz->frameQueue_->enqueue(picParams, thiz->currentFramePackets); + thiz->currentFramePackets.clear(); return true; } diff --git a/modules/cudacodec/src/video_parser.hpp b/modules/cudacodec/src/video_parser.hpp index 91e50b3e2b..870a2105a8 100644 --- a/modules/cudacodec/src/video_parser.hpp +++ b/modules/cudacodec/src/video_parser.hpp @@ -59,7 +59,7 @@ class VideoParser cuvidDestroyVideoParser(parser_); } - bool parseVideoData(const unsigned char* data, size_t size, bool endOfStream); + bool parseVideoData(const unsigned char* data, size_t size, const bool rawMode, const bool containsKeyFrame, bool endOfStream); bool hasError() const { return hasError_; } @@ -68,6 +68,7 @@ class VideoParser FrameQueue* frameQueue_; CUvideoparser parser_; int unparsedPackets_; + std::vector currentFramePackets; volatile bool hasError_; // Called when the decoder encounters a video format change (or initial sequence header) diff --git a/modules/cudacodec/src/video_reader.cpp b/modules/cudacodec/src/video_reader.cpp index b705e1618d..49884f17b2 100644 --- a/modules/cudacodec/src/video_reader.cpp +++ b/modules/cudacodec/src/video_reader.cpp @@ -48,8 +48,8 @@ using namespace cv::cudacodec; #ifndef HAVE_NVCUVID -Ptr cv::cudacodec::createVideoReader(const String&, const String = "", const bool = false) { throw_no_cuda(); return Ptr(); } -Ptr cv::cudacodec::createVideoReader(const Ptr&) { throw_no_cuda(); return Ptr(); } +Ptr cv::cudacodec::createVideoReader(const String&, const bool) { throw_no_cuda(); return Ptr(); } +Ptr cv::cudacodec::createVideoReader(const Ptr&, const bool) { throw_no_cuda(); return Ptr(); } #else // HAVE_NVCUVID @@ -69,18 +69,31 @@ namespace FormatInfo format() const CV_OVERRIDE; - void writeToFile(const std::string filename, const bool autoDetectExt = false) CV_OVERRIDE; + bool grab(Stream& stream) CV_OVERRIDE; + + bool retrieve(OutputArray frame, const size_t idx) const CV_OVERRIDE; + + bool set(const VideoReaderProps property, const double propertyVal) CV_OVERRIDE; + + int get(const VideoReaderProps property, const int propertyVal) const CV_OVERRIDE; private: + bool internalGrab(GpuMat& frame, Stream& stream); + Ptr videoSource_; Ptr frameQueue_ = 0; - Ptr videoDecoder_; - Ptr videoParser_; + Ptr videoDecoder_ = 0; + Ptr videoParser_ = 0; CUvideoctxlock lock_; std::deque< std::pair > frames_; + std::vector rawPackets; + GpuMat lastFrame; + static const int decodedFrameIdx = 0; + static const int extraDataIdx = 1; + static const int rawPacketsBaseIdx = 2; }; FormatInfo VideoReaderImpl::format() const @@ -122,18 +135,17 @@ namespace CUvideoctxlock m_lock; }; - bool VideoReaderImpl::nextFrame(GpuMat& frame, Stream& stream) - { + bool VideoReaderImpl::internalGrab(GpuMat& frame, Stream& stream) { if (videoSource_->hasError() || videoParser_->hasError()) CV_Error(Error::StsUnsupportedFormat, "Unsupported video source"); if (frames_.empty()) { CUVIDPARSERDISPINFO displayInfo; - + rawPackets.clear(); for (;;) { - if (frameQueue_->dequeue(displayInfo)) + if (frameQueue_->dequeue(displayInfo, rawPackets)) break; if (videoSource_->hasError() || videoParser_->hasError()) @@ -193,23 +205,89 @@ namespace return true; } - void VideoReaderImpl::writeToFile(const std::string filename, const bool autoDetectExt) + bool VideoReaderImpl::grab(Stream& stream) { + return internalGrab(lastFrame, stream); + } + + bool VideoReaderImpl::retrieve(OutputArray frame, const size_t idx) const { + if (idx == decodedFrameIdx) { + if (!frame.isGpuMat()) + CV_Error(Error::StsUnsupportedFormat, "Decoded frame is stored on the device and must be retrieved using a cv::cuda::GpuMat"); + frame.getGpuMatRef() = lastFrame; + } + else if (idx == extraDataIdx) { + if (!frame.isMat()) + CV_Error(Error::StsUnsupportedFormat, "Extra data is stored on the host and must be retrueved using a cv::Mat"); + videoSource_->getExtraData(frame.getMatRef()); + } + else{ + if (idx >= rawPacketsBaseIdx && idx < rawPacketsBaseIdx + rawPackets.size()) { + if (!frame.isMat()) + CV_Error(Error::StsUnsupportedFormat, "Raw data is stored on the host and must retrievd using a cv::Mat"); + Mat tmp(1, rawPackets.at(idx - rawPacketsBaseIdx).size, CV_8UC1, rawPackets.at(idx - rawPacketsBaseIdx).Data(), rawPackets.at(idx - rawPacketsBaseIdx).size); + frame.getMatRef() = tmp; + } + } + return !frame.empty(); + } + + bool VideoReaderImpl::set(const VideoReaderProps property, const double propertyVal) { + switch (property) { + case VideoReaderProps::PROP_RAW_MODE : + videoSource_->SetRawMode(static_cast(propertyVal)); + break; + } + return true; + } + + int VideoReaderImpl::get(const VideoReaderProps property, const int propertyVal) const { + switch (property) + { + case VideoReaderProps::PROP_DECODED_FRAME_IDX: + return decodedFrameIdx; + case VideoReaderProps::PROP_EXTRA_DATA_INDEX: + return extraDataIdx; + case VideoReaderProps::PROP_RAW_PACKAGES_BASE_INDEX: + if (videoSource_->RawModeEnabled()) + return rawPacketsBaseIdx; + else + break; + case VideoReaderProps::PROP_NUMBER_OF_RAW_PACKAGES_SINCE_LAST_GRAB: + return rawPackets.size(); + case::VideoReaderProps::PROP_RAW_MODE: + return videoSource_->RawModeEnabled(); + case::VideoReaderProps::PROP_LRF_HAS_KEY_FRAME: { + const int iPacket = propertyVal - rawPacketsBaseIdx; + if (videoSource_->RawModeEnabled() && iPacket >= 0 && iPacket < rawPackets.size()) + return rawPackets.at(iPacket).containsKeyFrame; + else + break; + } + default: + break; + } + return -1; + } + + bool VideoReaderImpl::nextFrame(GpuMat& frame, Stream& stream) { - return videoSource_->writeToFile(filename, autoDetectExt); + if (!internalGrab(frame, stream)) + return false; + return true; } } -Ptr cv::cudacodec::createVideoReader(const String& filename, const String filenameToWrite, const bool autoDetectExt) +Ptr cv::cudacodec::createVideoReader(const String& filename, const bool rawMode) { - CV_Assert( !filename.empty() ); + CV_Assert(!filename.empty()); Ptr videoSource; try { // prefer ffmpeg to cuvidGetSourceVideoFormat() which doesn't always return the corrct raw pixel format - Ptr source(new FFmpegVideoSource(filename, filenameToWrite, autoDetectExt)); - videoSource.reset(new RawVideoSourceWrapper(source)); + Ptr source(new FFmpegVideoSource(filename)); + videoSource.reset(new RawVideoSourceWrapper(source, rawMode)); } catch (...) { @@ -219,9 +297,9 @@ Ptr cv::cudacodec::createVideoReader(const String& filename, const return makePtr(videoSource); } -Ptr cv::cudacodec::createVideoReader(const Ptr& source) +Ptr cv::cudacodec::createVideoReader(const Ptr& source, const bool rawMode) { - Ptr videoSource(new RawVideoSourceWrapper(source)); + Ptr videoSource(new RawVideoSourceWrapper(source, rawMode)); return makePtr(videoSource); } diff --git a/modules/cudacodec/src/video_source.cpp b/modules/cudacodec/src/video_source.cpp index 36e1350f31..ab549fbd8e 100644 --- a/modules/cudacodec/src/video_source.cpp +++ b/modules/cudacodec/src/video_source.cpp @@ -49,14 +49,15 @@ using namespace cv; using namespace cv::cudacodec; using namespace cv::cudacodec::detail; -bool cv::cudacodec::detail::VideoSource::parseVideoData(const unsigned char* data, size_t size, bool endOfStream) +bool cv::cudacodec::detail::VideoSource::parseVideoData(const unsigned char* data, size_t size, const bool rawMode, const bool containsKeyFrame, bool endOfStream) { - return videoParser_->parseVideoData(data, size, endOfStream); + return videoParser_->parseVideoData(data, size, rawMode, containsKeyFrame, endOfStream); } -cv::cudacodec::detail::RawVideoSourceWrapper::RawVideoSourceWrapper(const Ptr& source) : +cv::cudacodec::detail::RawVideoSourceWrapper::RawVideoSourceWrapper(const Ptr& source, const bool rawMode) : source_(source) { + SetRawMode(rawMode); CV_Assert( !source_.empty() ); } @@ -70,11 +71,6 @@ void cv::cudacodec::detail::RawVideoSourceWrapper::updateFormat(const FormatInfo source_->updateFormat(videoFormat); } -void cv::cudacodec::detail::RawVideoSourceWrapper::writeToFile(const std::string filename, const bool autoDetectExt) -{ - source_->writeToFile(filename, autoDetectExt); -} - void cv::cudacodec::detail::RawVideoSourceWrapper::start() { stop_ = false; @@ -114,7 +110,19 @@ void cv::cudacodec::detail::RawVideoSourceWrapper::readLoop(void* userData) break; } - if (!thiz->parseVideoData(data, size)) + bool containsKeyFrame = false; + if (thiz->RawModeEnabled()) { + containsKeyFrame = thiz->source_->lastPacketContainsKeyFrame(); + if (!thiz->extraDataQueried) { + thiz->extraDataQueried = true; + Mat extraData; + thiz->source_->getExtraData(extraData); + if(!extraData.empty()) + thiz->setExtraData(extraData); + } + } + + if (!thiz->parseVideoData(data, size, thiz->RawModeEnabled(), containsKeyFrame)) { thiz->hasError_ = true; break; @@ -124,7 +132,7 @@ void cv::cudacodec::detail::RawVideoSourceWrapper::readLoop(void* userData) break; } - thiz->parseVideoData(0, 0, true); + thiz->parseVideoData(0, 0, false, false, true); } #endif // HAVE_NVCUVID diff --git a/modules/cudacodec/src/video_source.hpp b/modules/cudacodec/src/video_source.hpp index 125156efdc..ebaa3ba301 100644 --- a/modules/cudacodec/src/video_source.hpp +++ b/modules/cudacodec/src/video_source.hpp @@ -57,42 +57,48 @@ class VideoSource virtual FormatInfo format() const = 0; virtual void updateFormat(const FormatInfo& videoFormat) = 0; - virtual void writeToFile(const std::string filename, const bool autoDetectExt) = 0; virtual void start() = 0; virtual void stop() = 0; virtual bool isStarted() const = 0; virtual bool hasError() const = 0; - void setVideoParser(detail::VideoParser* videoParser) { videoParser_ = videoParser; } - + void setExtraData(const cv::Mat _extraData) { + AutoLock autoLock(mtx_); + extraData = _extraData.clone(); + } + void getExtraData(cv::Mat& _extraData) { + AutoLock autoLock(mtx_); + _extraData = extraData.clone(); + } + void SetRawMode(const bool enabled) { rawMode_ = enabled; } + bool RawModeEnabled() const { return rawMode_; } protected: - bool parseVideoData(const uchar* data, size_t size, bool endOfStream = false); - + bool parseVideoData(const uchar* data, size_t size, const bool rawMode, const bool containsKeyFrame, bool endOfStream = false); + bool extraDataQueried = false; private: - detail::VideoParser* videoParser_; + detail::VideoParser* videoParser_ = 0; + cv::Mat extraData; + bool rawMode_ = false; + Mutex mtx_; }; class RawVideoSourceWrapper : public VideoSource { public: - RawVideoSourceWrapper(const Ptr& source); + RawVideoSourceWrapper(const Ptr& source, const bool rawMode); FormatInfo format() const CV_OVERRIDE; void updateFormat(const FormatInfo& videoFormat) CV_OVERRIDE; - void writeToFile(const std::string filename, const bool autoDetectExt = false) CV_OVERRIDE; void start() CV_OVERRIDE; void stop() CV_OVERRIDE; bool isStarted() const CV_OVERRIDE; bool hasError() const CV_OVERRIDE; - private: - Ptr source_; - - Ptr thread_; + static void readLoop(void* userData); + Ptr source_ = 0; + Ptr thread_ = 0; volatile bool stop_; volatile bool hasError_; - - static void readLoop(void* userData); }; }}} diff --git a/modules/cudacodec/test/test_video.cpp b/modules/cudacodec/test/test_video.cpp index b4a586bea9..f23360dceb 100644 --- a/modules/cudacodec/test/test_video.cpp +++ b/modules/cudacodec/test/test_video.cpp @@ -41,16 +41,23 @@ //M*/ #include "test_precomp.hpp" -#include "opencv2/cudaarithm.hpp" namespace opencv_test { namespace { #if defined(HAVE_NVCUVID) || defined(HAVE_NVCUVENC) +PARAM_TEST_CASE(CheckSet, cv::cuda::DeviceInfo, std::string) +{ +}; + PARAM_TEST_CASE(Video, cv::cuda::DeviceInfo, std::string) { }; -PARAM_TEST_CASE(VideoReadWrite, cv::cuda::DeviceInfo, std::string) +PARAM_TEST_CASE(VideoReadRaw, cv::cuda::DeviceInfo, std::string) +{ +}; + +PARAM_TEST_CASE(CheckKeyFrame, cv::cuda::DeviceInfo, std::string) { }; @@ -58,6 +65,86 @@ PARAM_TEST_CASE(VideoReadWrite, cv::cuda::DeviceInfo, std::string) ////////////////////////////////////////////////////// // VideoReader +//========================================================================== + +CUDA_TEST_P(CheckSet, Reader) +{ + cv::cuda::setDevice(GET_PARAM(0).deviceID()); + + if (!videoio_registry::hasBackend(CAP_FFMPEG)) + throw SkipTestException("FFmpeg backend was not found"); + + std::string inputFile = std::string(cvtest::TS::ptr()->get_data_path()) + +"../" + GET_PARAM(1); + cv::Ptr reader = cv::cudacodec::createVideoReader(inputFile); + ASSERT_FALSE(reader->get(cv::cudacodec::VideoReaderProps::PROP_RAW_MODE)); + ASSERT_TRUE(reader->set(cv::cudacodec::VideoReaderProps::PROP_RAW_MODE,true)); + ASSERT_TRUE(reader->get(cv::cudacodec::VideoReaderProps::PROP_RAW_MODE)); + bool rawPacketsAvailable = false; + while (reader->grab()) { + if (reader->get(cv::cudacodec::VideoReaderProps::PROP_NUMBER_OF_RAW_PACKAGES_SINCE_LAST_GRAB) > 0) { + rawPacketsAvailable = true; + break; + } + } + ASSERT_TRUE(rawPacketsAvailable); +} + +typedef tuple check_extra_data_params_t; +typedef testing::TestWithParam< check_extra_data_params_t > CheckExtraData; + +CUDA_TEST_P(CheckExtraData, Reader) +{ + // RTSP streaming is only supported by the FFmpeg back end + if (!videoio_registry::hasBackend(CAP_FFMPEG)) + throw SkipTestException("FFmpeg backend not found"); + + const std::vector devices = cvtest::DeviceManager::instance().values(); + for (const auto& device : devices) { + cv::cuda::setDevice(device.deviceID()); + const string path = GET_PARAM(0); + const int sz = GET_PARAM(1); + std::string inputFile = std::string(cvtest::TS::ptr()->get_data_path()) + "../" + path; + cv::Ptr reader = cv::cudacodec::createVideoReader(inputFile, true); + ASSERT_TRUE(reader->get(cv::cudacodec::VideoReaderProps::PROP_RAW_MODE)); + const int extraDataIdx = reader->get(cv::cudacodec::VideoReaderProps::PROP_EXTRA_DATA_INDEX); + ASSERT_EQ(extraDataIdx, 1 ); + ASSERT_TRUE(reader->grab()); + cv::Mat extraData; + const bool newData = reader->retrieve(extraData, extraDataIdx); + ASSERT_TRUE(newData && sz || !newData && !sz); + ASSERT_EQ(extraData.total(), sz); + } +} + +CUDA_TEST_P(CheckKeyFrame, Reader) +{ + cv::cuda::setDevice(GET_PARAM(0).deviceID()); + + // RTSP streaming is only supported by the FFmpeg back end + if (!videoio_registry::hasBackend(CAP_FFMPEG)) + throw SkipTestException("FFmpeg backend not found"); + + const string path = GET_PARAM(1); + std::string inputFile = std::string(cvtest::TS::ptr()->get_data_path()) + "../" + path; + cv::Ptr reader = cv::cudacodec::createVideoReader(inputFile, true); + ASSERT_TRUE(reader->get(cv::cudacodec::VideoReaderProps::PROP_RAW_MODE)); + const int rawIdxBase = reader->get(cv::cudacodec::VideoReaderProps::PROP_RAW_PACKAGES_BASE_INDEX); + ASSERT_EQ(rawIdxBase, 2); + constexpr int maxNPackagesToCheck = 2; + int nPackages = 0; + while (nPackages < maxNPackagesToCheck) { + ASSERT_TRUE(reader->grab()); + const int N = reader->get(cv::cudacodec::VideoReaderProps::PROP_NUMBER_OF_RAW_PACKAGES_SINCE_LAST_GRAB); + for (int i = rawIdxBase; i < N + rawIdxBase; i++) { + nPackages++; + const bool containsKeyFrame = reader->get(cv::cudacodec::VideoReaderProps::PROP_LRF_HAS_KEY_FRAME, i); + ASSERT_TRUE(nPackages == 1 && containsKeyFrame || nPackages == 2 && !containsKeyFrame) << "nPackage: " << i; + if (nPackages >= maxNPackagesToCheck) + break; + } + } +} + CUDA_TEST_P(Video, Reader) { cv::cuda::setDevice(GET_PARAM(0).deviceID()); @@ -80,7 +167,7 @@ CUDA_TEST_P(Video, Reader) } } -CUDA_TEST_P(VideoReadWrite, Reader) +CUDA_TEST_P(VideoReadRaw, Reader) { cv::cuda::setDevice(GET_PARAM(0).deviceID()); @@ -91,13 +178,25 @@ CUDA_TEST_P(VideoReadWrite, Reader) std::string inputFile = std::string(cvtest::TS::ptr()->get_data_path()) + "../" + GET_PARAM(1); const string fileNameOut = tempfile("test_container_stream"); { - cv::Ptr reader = cv::cudacodec::createVideoReader(inputFile, fileNameOut); + std::ofstream file(fileNameOut, std::ios::binary); + ASSERT_TRUE(file.is_open()); + cv::Ptr reader = cv::cudacodec::createVideoReader(inputFile,true); + ASSERT_TRUE(reader->get(cv::cudacodec::VideoReaderProps::PROP_RAW_MODE)); + const int rawIdxBase = reader->get(cv::cudacodec::VideoReaderProps::PROP_RAW_PACKAGES_BASE_INDEX); + ASSERT_EQ(rawIdxBase, 2); cv::cuda::GpuMat frame; for (int i = 0; i < 100; i++) { - reader->writeToFile(fileNameOut.c_str()); - ASSERT_TRUE(reader->nextFrame(frame)); + ASSERT_TRUE(reader->grab()); + ASSERT_TRUE(reader->retrieve(frame)); ASSERT_FALSE(frame.empty()); + const int N = reader->get(cv::cudacodec::VideoReaderProps::PROP_NUMBER_OF_RAW_PACKAGES_SINCE_LAST_GRAB); + ASSERT_TRUE(N >= 0) << N << " < 0"; + for (int i = rawIdxBase; i <= N + rawIdxBase; i++) { + Mat rawPackets; + reader->retrieve(rawPackets, i); + file.write((char*)rawPackets.data, rawPackets.total()); + } } } @@ -105,13 +204,16 @@ CUDA_TEST_P(VideoReadWrite, Reader) { cv::Ptr readerReference = cv::cudacodec::createVideoReader(inputFile); - cv::Ptr readerActual = cv::cudacodec::createVideoReader(fileNameOut); + cv::Ptr readerActual = cv::cudacodec::createVideoReader(fileNameOut,true); + const int decodedFrameIdx = readerActual->get(cv::cudacodec::VideoReaderProps::PROP_DECODED_FRAME_IDX); + ASSERT_EQ(decodedFrameIdx, 0); cv::cuda::GpuMat reference, actual; cv::Mat referenceHost, actualHost; for (int i = 0; i < 100; i++) { ASSERT_TRUE(readerReference->nextFrame(reference)); - ASSERT_TRUE(readerActual->nextFrame(actual)); + ASSERT_TRUE(readerActual->grab()); + ASSERT_TRUE(readerActual->retrieve(actual, decodedFrameIdx)); actual.download(actualHost); reference.download(referenceHost); ASSERT_TRUE(cvtest::norm(actualHost, referenceHost, NORM_INF) == 0); @@ -171,6 +273,10 @@ CUDA_TEST_P(Video, Writer) #endif // _WIN32, HAVE_NVCUVENC +INSTANTIATE_TEST_CASE_P(CUDA_Codec, CheckSet, testing::Combine( + ALL_DEVICES, + testing::Values("highgui/video/big_buck_bunny.mp4"))); + #define VIDEO_SRC_R "highgui/video/big_buck_bunny.mp4", "cv/video/768x576.avi", "cv/video/1920x1080.avi", "highgui/video/big_buck_bunny.avi", \ "highgui/video/big_buck_bunny.h264", "highgui/video/big_buck_bunny.h265", "highgui/video/big_buck_bunny.mpg" INSTANTIATE_TEST_CASE_P(CUDA_Codec, Video, testing::Combine( @@ -178,10 +284,29 @@ INSTANTIATE_TEST_CASE_P(CUDA_Codec, Video, testing::Combine( testing::Values(VIDEO_SRC_R))); #define VIDEO_SRC_RW "highgui/video/big_buck_bunny.h264", "highgui/video/big_buck_bunny.h265" - -INSTANTIATE_TEST_CASE_P(CUDA_Codec, VideoReadWrite, testing::Combine( +INSTANTIATE_TEST_CASE_P(CUDA_Codec, VideoReadRaw, testing::Combine( ALL_DEVICES, testing::Values(VIDEO_SRC_RW))); +const check_extra_data_params_t check_extra_data_params[] = +{ + check_extra_data_params_t("cv/video/768x576.avi", 44), + check_extra_data_params_t("cv/video/1920x1080.avi", 47), + check_extra_data_params_t("highgui/video/big_buck_bunny.h264", 38), + check_extra_data_params_t("highgui/video/big_buck_bunny.h265", 84), + check_extra_data_params_t("highgui/video/big_buck_bunny.mp4", 45), + check_extra_data_params_t("highgui/video/big_buck_bunny.avi", 58), + check_extra_data_params_t("highgui/video/big_buck_bunny.mpg", 12), + check_extra_data_params_t("highgui/video/big_buck_bunny.mov", 45), + check_extra_data_params_t("highgui/video/big_buck_bunny.mjpg.avi", 0) +}; + +INSTANTIATE_TEST_CASE_P(CUDA_Codec, CheckExtraData, testing::ValuesIn(check_extra_data_params)); + +INSTANTIATE_TEST_CASE_P(CUDA_Codec, CheckKeyFrame, testing::Combine( + ALL_DEVICES, + testing::Values(VIDEO_SRC_R))); + + #endif // HAVE_NVCUVID || HAVE_NVCUVENC }} // namespace From c644e14f4b76e45c3da658ccc6c649823a9dbe19 Mon Sep 17 00:00:00 2001 From: cudawarped Date: Wed, 8 Dec 2021 16:54:22 +0000 Subject: [PATCH 9/9] Fix missing documentation. --- modules/cudacodec/include/opencv2/cudacodec.hpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/cudacodec/include/opencv2/cudacodec.hpp b/modules/cudacodec/include/opencv2/cudacodec.hpp index 61dec07961..7b8678df81 100644 --- a/modules/cudacodec/include/opencv2/cudacodec.hpp +++ b/modules/cudacodec/include/opencv2/cudacodec.hpp @@ -355,7 +355,7 @@ class CV_EXPORTS_W VideoReader /** @brief Returns previously grabbed video data. - @param [out] image The returned data which depends on the provided idx. If there is no new data since the last call to grab() the image will be empty. + @param [out] frame The returned data which depends on the provided idx. If there is no new data since the last call to grab() the image will be empty. @param idx Determins the returned data inside image. The returned data can be the: Decoded frame, idx = get(PROP_DECODED_FRAME_IDX). Extra data if available, idx = get(PROP_EXTRA_DATA_INDEX). @@ -430,6 +430,7 @@ CV_EXPORTS_W Ptr createVideoReader(const String& filename, const bo /** @overload @param source RAW video source implemented by user. +@param rawMode Allow the raw encoded data which has been read up until the last call to grab() to be retrieved by calling retrieve(rawData,RAW_DATA_IDX). */ CV_EXPORTS_W Ptr createVideoReader(const Ptr& source, const bool rawMode = false);