cudacodec: return luma hist from VideoReader::nextFrame if requested

Author: cudawarped

modules/cudacodec/include/opencv2/cudacodec.hpp

@@ -295,7 +295,7 @@ enum ChromaFormat
 
 /** @brief Deinterlacing mode used by decoder.
 * @param Weave Weave both fields (no deinterlacing). For progressive content and for content that doesn't need deinterlacing.
-* Bob Drop one field.
+* @param Bob Drop one field.
 * @param Adaptive Adaptive deinterlacing needs more video memory than other deinterlacing modes.
 * */
 enum DeinterlaceMode
@@ -305,12 +305,22 @@ enum DeinterlaceMode
     Adaptive = 2
 };
 
+/** @brief Utility function demonstrating how to map the luma histogram when FormatInfo::videoFullRangeFlag == false
+    @param hist Luma histogram \a hist returned from VideoReader::nextFrame(GpuMat& frame, GpuMat& hist, Stream& stream).
+    @param histFull Host histogram equivelent to downloading \a hist after calling cuda::calcHist(InputArray frame, OutputArray hist, Stream& stream).
+
+    @note
+    -   This function demonstrates how to map the luma histogram back so that it is equivalent to the result obtained from cuda::calcHist()
+    if the returned frame was colorFormat::GRAY.
+ */
+CV_EXPORTS_W void MapHist(const GpuMat& hist, CV_OUT Mat& histFull);
+
 /** @brief Struct providing information about video file format. :
  */
 struct CV_EXPORTS_W_SIMPLE FormatInfo
 {
     CV_WRAP FormatInfo() : nBitDepthMinus8(-1), ulWidth(0), ulHeight(0), width(0), height(0), ulMaxWidth(0), ulMaxHeight(0), valid(false),
-        fps(0), ulNumDecodeSurfaces(0), videoFullRangeFlag(false) {};
+        fps(0), ulNumDecodeSurfaces(0), videoFullRangeFlag(false), histogramEnabled(false), nCounterBitDepth(0), nMaxHistogramBins(0){};
 
     CV_PROP_RW Codec codec;
     CV_PROP_RW ChromaFormat chromaFormat;
@@ -330,6 +340,9 @@ struct CV_EXPORTS_W_SIMPLE FormatInfo
     CV_PROP_RW cv::Rect srcRoi;//!< Region of interest decoded from video source.
     CV_PROP_RW cv::Rect targetRoi;//!< Region of interest in the output frame containing the decoded frame.
     CV_PROP_RW bool videoFullRangeFlag;//!< Output value indicating if the black level, luma and chroma of the source are represented using the full or limited range (AKA TV or "analogue" range) of values as defined in Annex E of the ITU-T Specification.  Internally the conversion from NV12 to BGR obeys ITU 709.
+    CV_PROP_RW bool histogramEnabled;//!< Flag indicating whether histogram output has been enabled and is supported.
+    CV_PROP_RW int nCounterBitDepth;//!< Bit depth of histogram bins.
+    CV_PROP_RW int nMaxHistogramBins;//!< Max number of histogram bins.
 };
 
 /** @brief cv::cudacodec::VideoReader generic properties identifier.
@@ -375,6 +388,20 @@ class CV_EXPORTS_W VideoReader
      */
     CV_WRAP virtual bool nextFrame(CV_OUT GpuMat& frame, Stream &stream = Stream::Null()) = 0;
 
+    /** @brief Grabs, decodes and returns the next video frame and frame luma histogram.
+
+    @param [out] frame The video frame.
+    @param [out] histogram Histogram of the luma component of the encoded frame, see note.
+    @param stream Stream for the asynchronous version.
+    @return `false` if no frames have been grabbed.
+
+    If no frames have been grabbed (there are no more frames in video file), the methods return false.
+    The method throws an Exception if error occurs.
+
+    @note Histogram data is collected by NVDEC during the decoding process resulting in zero performance penalty. NVDEC computes the histogram data for only the luma component of decoded output, not on post-processed frame(i.e. when scaling, cropping, etc. applied).  If the source is encoded using a limited range of luma values (FormatInfo::videoFullRangeFlag == false) then the histogram bin values will correspond to to this limited range of values and will need to be mapped to contain the same output as cuda::calcHist().  The MapHist() utility function can be used to perform this mapping on the host if required.
+     */
+    CV_WRAP_AS(nextFrameWithHist) virtual bool nextFrame(CV_OUT GpuMat& frame, CV_OUT GpuMat& histogram, Stream& stream = Stream::Null()) = 0;
+
     /** @brief Returns information about video file format.
     */
     CV_WRAP virtual FormatInfo format() const = 0;
@@ -534,16 +561,18 @@ but it cannot go below the number determined by NVDEC.
 @param srcRoi Region of interest (x/width should be multiples of 4 and y/height multiples of 2) decoded from video source, defaults to the full frame.
 @param targetRoi Region of interest (x/width should be multiples of 4 and y/height multiples of 2) within the output frame to copy and resize the decoded frame to,
 defaults to the full frame.
+@param enableHistogramOutput Request output of decoded luma histogram \a hist from VideoReader::nextFrame(GpuMat& frame, GpuMat& hist, Stream& stream) if supported.
 */
 struct CV_EXPORTS_W_SIMPLE VideoReaderInitParams {
-    CV_WRAP VideoReaderInitParams() : udpSource(false), allowFrameDrop(false), minNumDecodeSurfaces(0), rawMode(0) {};
+    CV_WRAP VideoReaderInitParams() : udpSource(false), allowFrameDrop(false), minNumDecodeSurfaces(0), rawMode(0), enableHistogramOutput(false){};
     CV_PROP_RW bool udpSource;
     CV_PROP_RW bool allowFrameDrop;
     CV_PROP_RW int minNumDecodeSurfaces;
     CV_PROP_RW bool rawMode;
     CV_PROP_RW cv::Size targetSz;
     CV_PROP_RW cv::Rect srcRoi;
     CV_PROP_RW cv::Rect targetRoi;
+    CV_PROP_RW bool enableHistogramOutput;
 };
 
 /** @brief Creates video reader.

modules/cudacodec/misc/python/test/test_cudacodec.py

@@ -14,43 +14,61 @@ def setUp(self):
     @unittest.skipIf('OPENCV_TEST_DATA_PATH' not in os.environ,
                      "OPENCV_TEST_DATA_PATH is not defined")
     def test_reader(self):
-        #Test the functionality but not the results of the video reader
+        # Test the functionality but not the results of the VideoReader
 
-        vid_path = os.environ['OPENCV_TEST_DATA_PATH'] + '/cv/video/1920x1080.avi'
+        vid_path = os.environ['OPENCV_TEST_DATA_PATH'] + '/highgui/video/big_buck_bunny.h264'
         try:
             reader = cv.cudacodec.createVideoReader(vid_path)
             format_info = reader.format()
             ret, gpu_mat = reader.nextFrame()
             self.assertTrue(ret)
-            self.assertTrue('GpuMat' in str(type(gpu_mat)), msg=type(gpu_mat))
+            self.assertTrue(isinstance(gpu_mat, cv.cuda.GpuMat), msg=type(gpu_mat))
             #TODO: print(cv.utils.dumpInputArray(gpu_mat)) # - no support for GpuMat
 
+            # Retrieve format info
             if(not format_info.valid):
               format_info = reader.format()
             sz = gpu_mat.size()
             self.assertTrue(sz[0] == format_info.width and sz[1] == format_info.height)
 
             # not checking output, therefore sepearate tests for different signatures is unecessary
-            ret, _gpu_mat2 = reader.nextFrame(gpu_mat)
-            #TODO: self.assertTrue(gpu_mat == gpu_mat2)
-            self.assertTrue(ret)
+            ret, gpu_mat_ = reader.nextFrame(gpu_mat)
+            self.assertTrue(ret and gpu_mat_.cudaPtr() == gpu_mat.cudaPtr())
 
+            # Pass VideoReaderInitParams to the decoder and initialization params to the source (cv::VideoCapture)
             params = cv.cudacodec.VideoReaderInitParams()
             params.rawMode = True
+            params.enableHistogramOutput = True
             ms_gs = 1234
+            post_processed_sz = (gpu_mat.size()[0]*2, gpu_mat.size()[1]*2)
+            params.targetSz = post_processed_sz
             reader = cv.cudacodec.createVideoReader(vid_path,[cv.CAP_PROP_OPEN_TIMEOUT_MSEC, ms_gs], params)
             ret, ms = reader.get(cv.CAP_PROP_OPEN_TIMEOUT_MSEC)
             self.assertTrue(ret and ms == ms_gs)
             ret, raw_mode = reader.getVideoReaderProps(cv.cudacodec.VideoReaderProps_PROP_RAW_MODE)
             self.assertTrue(ret and raw_mode)
 
+            # Retrieve image histogram
+            ret, gpu_mat, hist = reader.nextFrameWithHist()
+            self.assertTrue(ret and not gpu_mat.empty() and hist.size() == (256,1))
+            ret, gpu_mat_, hist_ = reader.nextFrameWithHist(gpu_mat, hist)
+            self.assertTrue(ret and not gpu_mat.empty() and hist.size() == (256,1))
+            self.assertTrue(gpu_mat_.cudaPtr() == gpu_mat.cudaPtr() and hist_.cudaPtr() == hist.cudaPtr())
+            hist_host = cv.cudacodec.MapHist(hist)
+            self.assertTrue(hist_host.shape == (1,256) and isinstance(hist_host, np.ndarray))
+
+            # Check post processing applied
+            self.assertTrue(gpu_mat.size() == post_processed_sz)
+
+            # Change color format
             ret, colour_code = reader.getVideoReaderProps(cv.cudacodec.VideoReaderProps_PROP_COLOR_FORMAT)
             self.assertTrue(ret and colour_code == cv.cudacodec.ColorFormat_BGRA)
             colour_code_gs = cv.cudacodec.ColorFormat_GRAY
             reader.set(colour_code_gs)
             ret, colour_code = reader.getVideoReaderProps(cv.cudacodec.VideoReaderProps_PROP_COLOR_FORMAT)
             self.assertTrue(ret and colour_code == colour_code_gs)
 
+            # Read raw encoded bitstream
             ret, i_base = reader.getVideoReaderProps(cv.cudacodec.VideoReaderProps_PROP_RAW_PACKAGES_BASE_INDEX)
             self.assertTrue(ret and i_base == 2.0)
             self.assertTrue(reader.grab())
@@ -75,8 +93,8 @@ def test_reader(self):
             else:
                 self.skipTest(e.err)
 
-    def test_writer_existence(self):
-        #Test at least the existence of wrapped functions for now
+    def test_writer(self):
+        # Test the functionality but not the results of the VideoWriter
 
         try:
             fd, fname = tempfile.mkstemp(suffix=".h264")
@@ -91,11 +109,12 @@ def test_writer_existence(self):
             writer.write(blankFrameIn)
             writer.release()
             encoder_params_out = writer.getEncoderParams()
-            self.assert_true(encoder_params_in.gopLength == encoder_params_out.gopLength)
+            self.assertTrue(encoder_params_in.gopLength == encoder_params_out.gopLength)
             cap = cv.VideoCapture(fname,cv.CAP_FFMPEG)
-            self.assert_true(cap.isOpened())
+            self.assertTrue(cap.isOpened())
             ret, blankFrameOut = cap.read()
-            self.assert_true(ret and blankFrameOut.shape == blankFrameIn.download().shape)
+            self.assertTrue(ret and blankFrameOut.shape == blankFrameIn.download().shape)
+            cap.release()
         except cv.error as e:
             self.assertEqual(e.code, cv.Error.StsNotImplemented)
             self.skipTest("Either NVCUVENC or a GPU hardware encoder is missing or the encoding profile is not supported.")

modules/cudacodec/src/video_decoder.cpp

@@ -62,12 +62,8 @@ static const char* GetVideoChromaFormatString(cudaVideoChromaFormat eChromaForma
     return "Unknown";
 }
 
-void cv::cudacodec::detail::VideoDecoder::create(const FormatInfo& videoFormat)
+void cv::cudacodec::detail::VideoDecoder::create(FormatInfo& videoFormat)
 {
-    {
-        AutoLock autoLock(mtx_);
-        videoFormat_ = videoFormat;
-    }
     const cudaVideoCodec _codec = static_cast<cudaVideoCodec>(videoFormat.codec);
     const cudaVideoChromaFormat _chromaFormat = static_cast<cudaVideoChromaFormat>(videoFormat.chromaFormat);
     if (videoFormat.nBitDepthMinus8 > 0) {
@@ -127,15 +123,41 @@ void cv::cudacodec::detail::VideoDecoder::create(const FormatInfo& videoFormat)
         CV_Error(Error::StsUnsupportedFormat, "Video source is not supported by hardware video decoder");
         CV_LOG_ERROR(NULL, "Video source is not supported by hardware video decoder.");
     }
+
+    if (videoFormat.histogramEnabled){
+        if (decodeCaps.bIsHistogramSupported) {
+            if (decodeCaps.nCounterBitDepth != 32) {
+                std::ostringstream warning;
+                warning << "Luma histogram output disabled due to current device using " << decodeCaps.nCounterBitDepth << " bit bins.  Histogram output only supports 32 bit bins.";
+                CV_LOG_INFO(NULL, warning.str());
+                videoFormat.histogramEnabled = false;
+            }
+            else {
+                videoFormat.histogramEnabled = true;
+                videoFormat.nCounterBitDepth = decodeCaps.nCounterBitDepth;
+                videoFormat.nMaxHistogramBins = decodeCaps.nMaxHistogramBins;
+            }
+        }
+        else {
+            CV_LOG_INFO(NULL, "Luma histogram output is not supported for current codec and/or on current device.");
+            videoFormat.histogramEnabled = false;
+        }
+    }
+
     CV_Assert(videoFormat.ulWidth >= decodeCaps.nMinWidth &&
         videoFormat.ulHeight >= decodeCaps.nMinHeight &&
         videoFormat.ulWidth <= decodeCaps.nMaxWidth &&
         videoFormat.ulHeight <= decodeCaps.nMaxHeight);
 
     CV_Assert((videoFormat.width >> 4)* (videoFormat.height >> 4) <= decodeCaps.nMaxMBCount);
 #endif
+    {
+        AutoLock autoLock(mtx_);
+        videoFormat_ = videoFormat;
+    }
     // Create video decoder
     CUVIDDECODECREATEINFO createInfo_ = {};
+    createInfo_.enableHistogram = videoFormat.histogramEnabled;
     createInfo_.CodecType           = _codec;
     createInfo_.ulWidth             = videoFormat.ulWidth;
     createInfo_.ulHeight            = videoFormat.ulHeight;

modules/cudacodec/src/video_decoder.hpp

@@ -49,11 +49,12 @@ namespace cv { namespace cudacodec { namespace detail {
 class VideoDecoder
 {
 public:
-    VideoDecoder(const Codec& codec, const int minNumDecodeSurfaces, cv::Size targetSz, cv::Rect srcRoi, cv::Rect targetRoi, CUcontext ctx, CUvideoctxlock lock) :
+    VideoDecoder(const Codec& codec, const int minNumDecodeSurfaces, cv::Size targetSz, cv::Rect srcRoi, cv::Rect targetRoi, const bool enableHistogramOutput, CUcontext ctx, CUvideoctxlock lock) :
         ctx_(ctx), lock_(lock), decoder_(0)
     {
         videoFormat_.codec = codec;
         videoFormat_.ulNumDecodeSurfaces = minNumDecodeSurfaces;
+        videoFormat_.histogramEnabled = enableHistogramOutput;
         // alignment enforced by nvcuvid, likely due to chroma subsampling
         videoFormat_.targetSz.width = targetSz.width - targetSz.width % 2; videoFormat_.targetSz.height = targetSz.height - targetSz.height % 2;
         videoFormat_.srcRoi.x = srcRoi.x - srcRoi.x % 4; videoFormat_.srcRoi.width = srcRoi.width - srcRoi.width % 4;
@@ -67,7 +68,7 @@ class VideoDecoder
         release();
     }
 
-    void create(const FormatInfo& videoFormat);
+    void create(FormatInfo& videoFormat);
     void release();
 
     // Get the code-type currently used.
@@ -86,13 +87,14 @@ class VideoDecoder
 
     cudaVideoChromaFormat chromaFormat() const { return static_cast<cudaVideoChromaFormat>(videoFormat_.chromaFormat); }
     int nBitDepthMinus8() const { return videoFormat_.nBitDepthMinus8; }
+    bool histogramEnabled() const { return videoFormat_.histogramEnabled; }
 
     bool decodePicture(CUVIDPICPARAMS* picParams)
     {
         return cuvidDecodePicture(decoder_, picParams) == CUDA_SUCCESS;
     }
 
-    cuda::GpuMat mapFrame(int picIdx, CUVIDPROCPARAMS& videoProcParams)
+    GpuMat mapFrame(int picIdx, CUVIDPROCPARAMS& videoProcParams)
     {
         CUdeviceptr ptr;
         unsigned int pitch;
@@ -102,6 +104,8 @@ class VideoDecoder
         return cuda::GpuMat(targetHeight() * 3 / 2, targetWidth(), CV_8UC1, (void*) ptr, pitch);
     }
 
+
+
     void unmapFrame(cuda::GpuMat& frame)
     {
         cuSafeCall( cuvidUnmapVideoFrame(decoder_, (CUdeviceptr) frame.data) );

modules/cudacodec/src/video_parser.cpp

@@ -154,6 +154,7 @@ int CUDAAPI cv::cudacodec::detail::VideoParser::HandleVideoSequence(void* userDa
             maxH = format->coded_height;
         newFormat.ulMaxWidth = maxW;
         newFormat.ulMaxHeight = maxH;
+        newFormat.histogramEnabled = thiz->videoDecoder_->histogramEnabled();
         thiz->frameQueue_->init(newFormat.ulNumDecodeSurfaces);
         try
         {