NVIDIA_OPTICAL_FLOW_2_0_INTEGRATION

Author: vchiluka5

modules/cudaoptflow/CMakeLists.txt

@@ -8,21 +8,21 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127 /wd4324 /wd4512 -Wundef -Wmissing-d
 
 ocv_define_module(cudaoptflow opencv_video opencv_optflow opencv_cudaarithm opencv_cudawarping opencv_cudaimgproc OPTIONAL opencv_cudalegacy WRAP python)
 
-set(NVIDIA_OPTICAL_FLOW_1_0_HEADERS_COMMIT "79c6cee80a2df9a196f20afd6b598a9810964c32")
-set(NVIDIA_OPTICAL_FLOW_1_0_HEADERS_MD5 "ca5acedee6cb45d0ec610a6732de5c15")
-set(NVIDIA_OPTICAL_FLOW_1_0_HEADERS_PATH "${OpenCV_BINARY_DIR}/3rdparty/NVIDIAOpticalFlowSDK_1_0_Headers")
-ocv_download(FILENAME "${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_COMMIT}.zip"
-               HASH ${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_MD5}
+set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT "edb50da3cf849840d680249aa6dbef248ebce2ca")
+set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5 "a73cd48b18dcc0cc8933b30796074191")
+set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH "${OpenCV_BINARY_DIR}/3rdparty/NVIDIAOpticalFlowSDK_2_0_Headers")
+ocv_download(FILENAME "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}.zip"
+               HASH ${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5}
                URL
                  "https://github.com/NVIDIA/NVIDIAOpticalFlowSDK/archive/"
-               DESTINATION_DIR "${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_PATH}"
-               STATUS NVIDIA_OPTICAL_FLOW_1_0_HEADERS_DOWNLOAD_SUCCESS
+               DESTINATION_DIR "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}"
+               STATUS NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS
                ID "NVIDIA_OPTICAL_FLOW"
                UNPACK RELATIVE_URL)
 
-if(NOT NVIDIA_OPTICAL_FLOW_1_0_HEADERS_DOWNLOAD_SUCCESS)
-  message(STATUS "Failed to download NVIDIA_Optical_Flow_1_0 Headers")
+if(NOT NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS)
+  message(STATUS "Failed to download NVIDIA_Optical_Flow_2_0 Headers")
 else()
   add_definitions(-DHAVE_NVIDIA_OPTFLOW=1)
-  ocv_include_directories(SYSTEM "${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_PATH}/NVIDIAOpticalFlowSDK-${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_COMMIT}")
+  ocv_include_directories(SYSTEM "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}/NVIDIAOpticalFlowSDK-${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}")
 endif()

modules/cudaoptflow/include/opencv2/cudaoptflow.hpp

@@ -392,9 +392,9 @@ class CV_EXPORTS_W OpticalFlowDual_TVL1 : public DenseOpticalFlow
 /** @brief Class for computing the optical flow vectors between two images using NVIDIA Optical Flow hardware and Optical Flow SDK 1.0.
 @note
 - A sample application demonstrating the use of NVIDIA Optical Flow can be found at
-opencv_source_code/samples/gpu/nvidia_optical_flow.cpp
+opencv_contrib_source_code/modules/cudaoptflow/samples/nvidia_optical_flow.cpp
 - An example application comparing accuracy and performance of NVIDIA Optical Flow with other optical flow algorithms in OpenCV can be found at
-opencv_source_code/samples/gpu/optical_flow.cpp
+opencv_contrib_source_code/modules/cudaoptflow/samples/optical_flow.cpp
 */
 
 class CV_EXPORTS_W NvidiaOpticalFlow_1_0 : public NvidiaHWOpticalFlow
@@ -417,18 +417,16 @@ class CV_EXPORTS_W NvidiaOpticalFlow_1_0 : public NvidiaHWOpticalFlow
     * using nearest neighbour upsampling method.
 
     @param flow Buffer of type CV_16FC2 containing flow vectors generated by calc().
-    @param width Width of the input image in pixels for which these flow vectors were generated.
-    @param height Height of the input image in pixels for which these flow vectors were generated.
+    @param imageSize Size of the input image in pixels for which these flow vectors were generated.
     @param gridSize Granularity of the optical flow vectors returned by calc() function. Can be queried using getGridSize().
     @param upsampledFlow Buffer of type CV_32FC2, containing upsampled flow vectors, each flow vector for 1 pixel, in the pitch-linear layout.
     */
-    CV_WRAP virtual void upSampler(InputArray flow, int width, int height,
+    CV_WRAP virtual void upSampler(InputArray flow, cv::Size imageSize,
         int gridSize, InputOutputArray upsampledFlow) = 0;
 
     /** @brief Instantiate NVIDIA Optical Flow
 
-    @param width Width of input image in pixels.
-    @param height Height of input image in pixels.
+    @param imageSize Size of input image in pixels.
     @param perfPreset Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about presets.
                       Defaults to NV_OF_PERF_LEVEL_SLOW.
     @param enableTemporalHints Optional parameter. Flag to enable temporal hints. When set to true, the hardware uses the flow vectors
@@ -445,10 +443,142 @@ class CV_EXPORTS_W NvidiaOpticalFlow_1_0 : public NvidiaHWOpticalFlow
                         If output stream is not set, the execute function will use default stream which is NULL stream;
     */
     CV_WRAP static Ptr<NvidiaOpticalFlow_1_0> create(
-        int width,
-        int height,
+        cv::Size imageSize,
         cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL perfPreset
-        = cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW,
+        = cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
+        bool enableTemporalHints = false,
+        bool enableExternalHints = false,
+        bool enableCostBuffer = false,
+        int gpuId = 0,
+        Stream& inputStream = Stream::Null(),
+        Stream& outputStream = Stream::Null());
+};
+
+/** @brief Class for computing the optical flow vectors between two images using NVIDIA Optical Flow hardware and Optical Flow SDK 2.0.
+@note
+- A sample application demonstrating the use of NVIDIA Optical Flow can be found at
+opencv_contrib_source_code/modules/cudaoptflow/samples/nvidia_optical_flow.cpp
+- An example application comparing accuracy and performance of NVIDIA Optical Flow with other optical flow algorithms in OpenCV can be found at
+opencv_contrib_source_code/modules/cudaoptflow/samples/optical_flow.cpp
+*/
+
+class CV_EXPORTS_W NvidiaOpticalFlow_2_0 : public NvidiaHWOpticalFlow
+{
+public:
+    /**
+    * Supported optical flow performance levels.
+    */
+    enum NVIDIA_OF_PERF_LEVEL
+    {
+        NV_OF_PERF_LEVEL_UNDEFINED,
+        NV_OF_PERF_LEVEL_SLOW = 5,                   /**< Slow perf level results in lowest performance and best quality */
+        NV_OF_PERF_LEVEL_MEDIUM = 10,                /**< Medium perf level results in low performance and medium quality */
+        NV_OF_PERF_LEVEL_FAST = 20,                  /**< Fast perf level results in high performance and low quality */
+        NV_OF_PERF_LEVEL_MAX
+    };
+
+    /**
+    * Supported grid size for output buffer.
+    */
+    enum NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE
+    {
+        NV_OF_OUTPUT_VECTOR_GRID_SIZE_UNDEFINED,
+        NV_OF_OUTPUT_VECTOR_GRID_SIZE_1 = 1,          /**< Output buffer grid size is 1x1  */
+        NV_OF_OUTPUT_VECTOR_GRID_SIZE_2 = 2,          /**< Output buffer grid size is 2x2  */
+        NV_OF_OUTPUT_VECTOR_GRID_SIZE_4 = 4,          /**< Output buffer grid size is 4x4  */
+        NV_OF_OUTPUT_VECTOR_GRID_SIZE_MAX
+    };
+
+    /**
+    * Supported grid size for hint buffer.
+    */
+    enum NVIDIA_OF_HINT_VECTOR_GRID_SIZE
+    {
+        NV_OF_HINT_VECTOR_GRID_SIZE_UNDEFINED,
+        NV_OF_HINT_VECTOR_GRID_SIZE_1 = 1,            /**< Hint buffer grid size is 1x1.*/
+        NV_OF_HINT_VECTOR_GRID_SIZE_2 = 2,            /**< Hint buffer grid size is 2x2.*/
+        NV_OF_HINT_VECTOR_GRID_SIZE_4 = 4,            /**< Hint buffer grid size is 4x4.*/
+        NV_OF_HINT_VECTOR_GRID_SIZE_8 = 8,            /**< Hint buffer grid size is 8x8.*/
+        NV_OF_HINT_VECTOR_GRID_SIZE_MAX
+    };
+
+    /** @brief convertToFloat() helper function converts the hardware-generated flow vectors to floating point representation (1 flow vector for gridSize).
+    * gridSize can be queried via function getGridSize().
+
+    @param flow Buffer of type CV_16FC2 containing flow vectors generated by calc().
+    @param floatFlow Buffer of type CV_32FC2, containing flow vectors in floating point representation, each flow vector for 1 pixel per gridSize, in the pitch-linear layout.
+    */
+    CV_WRAP virtual void convertToFloat(InputArray flow, InputOutputArray floatFlow) = 0;
+
+    /** @brief Instantiate NVIDIA Optical Flow
+
+    @param imageSize Size of input image in pixels.
+    @param perfPreset Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about presets.
+                      Defaults to NV_OF_PERF_LEVEL_SLOW.
+    @param outputGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about output grid sizes.
+                          Defaults to NV_OF_OUTPUT_VECTOR_GRID_SIZE_1.
+    @param hintGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about hint grid sizes.
+                        Defaults to NV_OF_HINT_VECTOR_GRID_SIZE_1.
+    @param enableTemporalHints Optional parameter. Flag to enable temporal hints. When set to true, the hardware uses the flow vectors
+                               generated in previous call to calc() as internal hints for the current call to calc().
+                               Useful when computing flow vectors between successive video frames. Defaults to false.
+    @param enableExternalHints Optional Parameter. Flag to enable passing external hints buffer to calc(). Defaults to false.
+    @param enableCostBuffer Optional Parameter. Flag to enable cost buffer output from calc(). Defaults to false.
+    @param gpuId Optional parameter to select the GPU ID on which the optical flow should be computed. Useful in multi-GPU systems. Defaults to 0.
+    @param inputStream Optical flow algorithm may optionally involve cuda preprocessing on the input buffers.
+                       The input cuda stream can be used to pipeline and synchronize the cuda preprocessing tasks with OF HW engine.
+                       If input stream is not set, the execute function will use default stream which is NULL stream;
+    @param outputStream Optical flow algorithm may optionally involve cuda post processing on the output flow vectors.
+                        The output cuda stream can be used to pipeline and synchronize the cuda post processing tasks with OF HW engine.
+                        If output stream is not set, the execute function will use default stream which is NULL stream;
+    */
+    CV_WRAP static Ptr<NvidiaOpticalFlow_2_0> create(
+        cv::Size imageSize,
+        cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL perfPreset
+        = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_PERF_LEVEL_SLOW,
+        cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize
+        = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1,
+        cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize
+        = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_HINT_VECTOR_GRID_SIZE_1,
+        bool enableTemporalHints = false,
+        bool enableExternalHints = false,
+        bool enableCostBuffer = false,
+        int gpuId = 0,
+        Stream& inputStream = Stream::Null(),
+        Stream& outputStream = Stream::Null());
+
+    /**  @brief Instantiate NVIDIA Optical Flow with ROI Feature
+
+    @param imageSize Size of input image in pixels.
+    @param roiData Pointer to ROI data.
+    @param perfPreset Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about presets.
+                      Defaults to NV_OF_PERF_LEVEL_SLOW.
+    @param outputGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about output grid sizes.
+                          Defaults to NV_OF_OUTPUT_VECTOR_GRID_SIZE_1.
+    @param hintGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about hint grid sizes.
+                        Defaults to NV_OF_HINT_VECTOR_GRID_SIZE_1.
+    @param enableTemporalHints Optional parameter. Flag to enable temporal hints. When set to true, the hardware uses the flow vectors
+                               generated in previous call to calc() as internal hints for the current call to calc().
+                               Useful when computing flow vectors between successive video frames. Defaults to false.
+    @param enableExternalHints Optional Parameter. Flag to enable passing external hints buffer to calc(). Defaults to false.
+    @param enableCostBuffer Optional Parameter. Flag to enable cost buffer output from calc(). Defaults to false.
+    @param gpuId Optional parameter to select the GPU ID on which the optical flow should be computed. Useful in multi-GPU systems. Defaults to 0.
+    @param inputStream Optical flow algorithm may optionally involve cuda preprocessing on the input buffers.
+                       The input cuda stream can be used to pipeline and synchronize the cuda preprocessing tasks with OF HW engine.
+                       If input stream is not set, the execute function will use default stream which is NULL stream;
+    @param outputStream Optical flow algorithm may optionally involve cuda post processing on the output flow vectors.
+                        The output cuda stream can be used to pipeline and synchronize the cuda post processing tasks with OF HW engine.
+                        If output stream is not set, the execute function will use default stream which is NULL stream;
+    */
+    CV_WRAP static Ptr<NvidiaOpticalFlow_2_0> create(
+        cv::Size imageSize,
+        std::vector<Rect> roiData,
+        cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL perfPreset
+        = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_PERF_LEVEL_SLOW,
+        cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize
+        = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1,
+        cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize
+        = cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_HINT_VECTOR_GRID_SIZE_1,
         bool enableTemporalHints = false,
         bool enableExternalHints = false,
         bool enableCostBuffer = false,

modules/cudaoptflow/misc/python/test/test_nvidiaopticalflow.py

@@ -0,0 +1,36 @@
+import os
+import cv2 as cv
+import numpy as np
+
+from tests_common import NewOpenCVTests, unittest
+
+class nvidiaopticalflow_test(NewOpenCVTests):
+    def setUp(self):
+        super(nvidiaopticalflow_test, self).setUp()
+        if not cv.cuda.getCudaEnabledDeviceCount():
+            self.skipTest("No CUDA-capable device is detected")
+
+    @unittest.skipIf('OPENCV_TEST_DATA_PATH' not in os.environ,
+                        "OPENCV_TEST_DATA_PATH is not defined")
+    def test_calc(self):
+        frame1 = os.environ['OPENCV_TEST_DATA_PATH'] + '/gpu/opticalflow/frame0.png'
+        frame2 = os.environ['OPENCV_TEST_DATA_PATH'] + '/gpu/opticalflow/frame1.png'
+
+        npMat1 = cv.cvtColor(cv.imread(frame1),cv.COLOR_BGR2GRAY)
+        npMat2 = cv.cvtColor(cv.imread(frame2),cv.COLOR_BGR2GRAY)
+
+        cuMat1 = cv.cuda_GpuMat(npMat1)
+        cuMat2 = cv.cuda_GpuMat(npMat2)
+        try:
+            nvof = cv.cuda_NvidiaOpticalFlow_1_0.create(cuMat1.shape[1], cuMat1.shape[0], 5, False, False, False, 0)
+            flow = nvof.calc(cuMat1, cuMat2, None)
+            self.assertTrue(flow.shape[1] > 0 and flow.shape[0] > 0)
+            flowUpSampled = nvof.upSampler(flow[0], cuMat1.shape[1], cuMat1.shape[0], nvof.getGridSize(), None)
+            nvof.collectGarbage()
+        except cv.error as e:
+            if e.code == cv.Error.StsBadFunc or e.code == cv.Error.StsBadArg or e.code == cv.Error.StsNullPtr:
+                self.skipTest("Algorithm is not supported in the current environment")
+        self.assertTrue(flowUpSampled.shape[1] > 0 and flowUpSampled.shape[0] > 0)
+
+if __name__ == '__main__':
+    NewOpenCVTests.bootstrap()

modules/cudaoptflow/perf/perf_optflow.cpp

@@ -339,13 +339,8 @@ PERF_TEST_P(ImagePair, NvidiaOpticalFlow_1_0,
 
     const cv::Mat frame1 = readImage(GetParam().second, cv::IMREAD_GRAYSCALE);
     ASSERT_FALSE(frame1.empty());
-
-    const int width = frame0.size().width;
-    const int height = frame0.size().height;
-    const bool enableTemporalHints = false;
-    const bool enableExternalHints = false;
-    const bool enableCostBuffer = false;
-    const int gpuid = 0;
+    Stream inputStream;
+    Stream outputStream;
 
     if (PERF_RUN_CUDA())
     {
@@ -355,9 +350,9 @@ PERF_TEST_P(ImagePair, NvidiaOpticalFlow_1_0,
         cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> d_nvof;
         try
         {
-            d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(width, height,
+            d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(frame0.size(),
                 cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST,
-                enableTemporalHints, enableExternalHints, enableCostBuffer, gpuid);
+                false, false, false, 0, inputStream, outputStream);
         }
         catch (const cv::Exception& e)
         {
@@ -376,6 +371,63 @@ PERF_TEST_P(ImagePair, NvidiaOpticalFlow_1_0,
 
         CUDA_SANITY_CHECK(u, 1e-10);
         CUDA_SANITY_CHECK(v, 1e-10);
+
+        d_nvof->collectGarbage();
+    }
+}
+
+//////////////////////////////////////////////////////
+// NvidiaOpticalFlow_2_0
+
+PERF_TEST_P(ImagePair, NvidiaOpticalFlow_2_0,
+    Values<pair_string>(make_pair("gpu/opticalflow/frame0.png", "gpu/opticalflow/frame1.png")))
+{
+    declare.time(10);
+
+    const cv::Mat frame0 = readImage(GetParam().first, cv::IMREAD_GRAYSCALE);
+    ASSERT_FALSE(frame0.empty());
+
+    const cv::Mat frame1 = readImage(GetParam().second, cv::IMREAD_GRAYSCALE);
+    ASSERT_FALSE(frame1.empty());
+
+    const cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outGridSize
+        = cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1;
+    const cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize
+        = cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE::NV_OF_HINT_VECTOR_GRID_SIZE_1;
+    Stream inputStream;
+    Stream outputStream;
+
+    if (PERF_RUN_CUDA())
+    {
+        const cv::cuda::GpuMat d_frame0(frame0);
+        const cv::cuda::GpuMat d_frame1(frame1);
+        cv::cuda::GpuMat d_flow;
+        cv::Ptr<cv::cuda::NvidiaOpticalFlow_2_0> d_nvof;
+        try
+        {
+            d_nvof = cv::cuda::NvidiaOpticalFlow_2_0::create(frame0.size(),
+                cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST, outGridSize, hintGridSize,
+              false, false, false, 0, inputStream, outputStream);
+        }
+        catch (const cv::Exception& e)
+        {
+            if (e.code == Error::StsBadFunc || e.code == Error::StsBadArg || e.code == Error::StsNullPtr)
+                throw SkipTestException("Current configuration is not supported");
+            throw;
+        }
+
+        TEST_CYCLE() d_nvof->calc(d_frame0, d_frame1, d_flow);
+
+        cv::cuda::GpuMat flow[2];
+        cv::cuda::split(d_flow, flow);
+
+        cv::cuda::GpuMat u = flow[0];
+        cv::cuda::GpuMat v = flow[1];
+
+        CUDA_SANITY_CHECK(u, 1e-10);
+        CUDA_SANITY_CHECK(v, 1e-10);
+
+        d_nvof->collectGarbage();
     }
 }