Skip to content

NVIDIA_OPTICAL_FLOW_2_0_INTEGRATION #2807

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jan 16, 2021
Merged

NVIDIA_OPTICAL_FLOW_2_0_INTEGRATION #2807

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 10 additions & 10 deletions modules/cudaoptflow/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,21 +8,21 @@ ocv_warnings_disable(CMAKE_CXX_FLAGS /wd4127 /wd4324 /wd4512 -Wundef -Wmissing-d

ocv_define_module(cudaoptflow opencv_video opencv_optflow opencv_cudaarithm opencv_cudawarping opencv_cudaimgproc OPTIONAL opencv_cudalegacy WRAP python)

set(NVIDIA_OPTICAL_FLOW_1_0_HEADERS_COMMIT "79c6cee80a2df9a196f20afd6b598a9810964c32")
set(NVIDIA_OPTICAL_FLOW_1_0_HEADERS_MD5 "ca5acedee6cb45d0ec610a6732de5c15")
set(NVIDIA_OPTICAL_FLOW_1_0_HEADERS_PATH "${OpenCV_BINARY_DIR}/3rdparty/NVIDIAOpticalFlowSDK_1_0_Headers")
ocv_download(FILENAME "${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_COMMIT}.zip"
HASH ${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_MD5}
set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT "edb50da3cf849840d680249aa6dbef248ebce2ca")
set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5 "a73cd48b18dcc0cc8933b30796074191")
set(NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH "${OpenCV_BINARY_DIR}/3rdparty/NVIDIAOpticalFlowSDK_2_0_Headers")
ocv_download(FILENAME "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}.zip"
HASH ${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_MD5}
URL
"https://github.com/NVIDIA/NVIDIAOpticalFlowSDK/archive/"
DESTINATION_DIR "${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_PATH}"
STATUS NVIDIA_OPTICAL_FLOW_1_0_HEADERS_DOWNLOAD_SUCCESS
DESTINATION_DIR "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}"
STATUS NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS
ID "NVIDIA_OPTICAL_FLOW"
UNPACK RELATIVE_URL)

if(NOT NVIDIA_OPTICAL_FLOW_1_0_HEADERS_DOWNLOAD_SUCCESS)
message(STATUS "Failed to download NVIDIA_Optical_Flow_1_0 Headers")
if(NOT NVIDIA_OPTICAL_FLOW_2_0_HEADERS_DOWNLOAD_SUCCESS)
message(STATUS "Failed to download NVIDIA_Optical_Flow_2_0 Headers")
else()
add_definitions(-DHAVE_NVIDIA_OPTFLOW=1)
ocv_include_directories(SYSTEM "${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_PATH}/NVIDIAOpticalFlowSDK-${NVIDIA_OPTICAL_FLOW_1_0_HEADERS_COMMIT}")
ocv_include_directories(SYSTEM "${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_PATH}/NVIDIAOpticalFlowSDK-${NVIDIA_OPTICAL_FLOW_2_0_HEADERS_COMMIT}")
endif()
150 changes: 140 additions & 10 deletions modules/cudaoptflow/include/opencv2/cudaoptflow.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -392,9 +392,9 @@ class CV_EXPORTS_W OpticalFlowDual_TVL1 : public DenseOpticalFlow
/** @brief Class for computing the optical flow vectors between two images using NVIDIA Optical Flow hardware and Optical Flow SDK 1.0.
@note
- A sample application demonstrating the use of NVIDIA Optical Flow can be found at
opencv_source_code/samples/gpu/nvidia_optical_flow.cpp
opencv_contrib_source_code/modules/cudaoptflow/samples/nvidia_optical_flow.cpp
- An example application comparing accuracy and performance of NVIDIA Optical Flow with other optical flow algorithms in OpenCV can be found at
opencv_source_code/samples/gpu/optical_flow.cpp
opencv_contrib_source_code/modules/cudaoptflow/samples/optical_flow.cpp
*/

class CV_EXPORTS_W NvidiaOpticalFlow_1_0 : public NvidiaHWOpticalFlow
Expand All @@ -417,18 +417,16 @@ class CV_EXPORTS_W NvidiaOpticalFlow_1_0 : public NvidiaHWOpticalFlow
* using nearest neighbour upsampling method.

@param flow Buffer of type CV_16FC2 containing flow vectors generated by calc().
@param width Width of the input image in pixels for which these flow vectors were generated.
@param height Height of the input image in pixels for which these flow vectors were generated.
@param imageSize Size of the input image in pixels for which these flow vectors were generated.
@param gridSize Granularity of the optical flow vectors returned by calc() function. Can be queried using getGridSize().
@param upsampledFlow Buffer of type CV_32FC2, containing upsampled flow vectors, each flow vector for 1 pixel, in the pitch-linear layout.
*/
CV_WRAP virtual void upSampler(InputArray flow, int width, int height,
CV_WRAP virtual void upSampler(InputArray flow, cv::Size imageSize,
int gridSize, InputOutputArray upsampledFlow) = 0;

/** @brief Instantiate NVIDIA Optical Flow

@param width Width of input image in pixels.
@param height Height of input image in pixels.
@param imageSize Size of input image in pixels.
@param perfPreset Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about presets.
Defaults to NV_OF_PERF_LEVEL_SLOW.
@param enableTemporalHints Optional parameter. Flag to enable temporal hints. When set to true, the hardware uses the flow vectors
Expand All @@ -445,10 +443,142 @@ class CV_EXPORTS_W NvidiaOpticalFlow_1_0 : public NvidiaHWOpticalFlow
If output stream is not set, the execute function will use default stream which is NULL stream;
*/
CV_WRAP static Ptr<NvidiaOpticalFlow_1_0> create(
int width,
int height,
cv::Size imageSize,
cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL perfPreset
= cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_SLOW,
= cv::cuda::NvidiaOpticalFlow_1_0::NV_OF_PERF_LEVEL_SLOW,
bool enableTemporalHints = false,
bool enableExternalHints = false,
bool enableCostBuffer = false,
int gpuId = 0,
Stream& inputStream = Stream::Null(),
Stream& outputStream = Stream::Null());
};

/** @brief Class for computing the optical flow vectors between two images using NVIDIA Optical Flow hardware and Optical Flow SDK 2.0.
@note
- A sample application demonstrating the use of NVIDIA Optical Flow can be found at
opencv_contrib_source_code/modules/cudaoptflow/samples/nvidia_optical_flow.cpp
- An example application comparing accuracy and performance of NVIDIA Optical Flow with other optical flow algorithms in OpenCV can be found at
opencv_contrib_source_code/modules/cudaoptflow/samples/optical_flow.cpp
*/

class CV_EXPORTS_W NvidiaOpticalFlow_2_0 : public NvidiaHWOpticalFlow
{
public:
/**
* Supported optical flow performance levels.
*/
enum NVIDIA_OF_PERF_LEVEL
{
NV_OF_PERF_LEVEL_UNDEFINED,
NV_OF_PERF_LEVEL_SLOW = 5, /**< Slow perf level results in lowest performance and best quality */
NV_OF_PERF_LEVEL_MEDIUM = 10, /**< Medium perf level results in low performance and medium quality */
NV_OF_PERF_LEVEL_FAST = 20, /**< Fast perf level results in high performance and low quality */
NV_OF_PERF_LEVEL_MAX
};

/**
* Supported grid size for output buffer.
*/
enum NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE
{
NV_OF_OUTPUT_VECTOR_GRID_SIZE_UNDEFINED,
NV_OF_OUTPUT_VECTOR_GRID_SIZE_1 = 1, /**< Output buffer grid size is 1x1 */
NV_OF_OUTPUT_VECTOR_GRID_SIZE_2 = 2, /**< Output buffer grid size is 2x2 */
NV_OF_OUTPUT_VECTOR_GRID_SIZE_4 = 4, /**< Output buffer grid size is 4x4 */
NV_OF_OUTPUT_VECTOR_GRID_SIZE_MAX
};

/**
* Supported grid size for hint buffer.
*/
enum NVIDIA_OF_HINT_VECTOR_GRID_SIZE
{
NV_OF_HINT_VECTOR_GRID_SIZE_UNDEFINED,
NV_OF_HINT_VECTOR_GRID_SIZE_1 = 1, /**< Hint buffer grid size is 1x1.*/
NV_OF_HINT_VECTOR_GRID_SIZE_2 = 2, /**< Hint buffer grid size is 2x2.*/
NV_OF_HINT_VECTOR_GRID_SIZE_4 = 4, /**< Hint buffer grid size is 4x4.*/
NV_OF_HINT_VECTOR_GRID_SIZE_8 = 8, /**< Hint buffer grid size is 8x8.*/
NV_OF_HINT_VECTOR_GRID_SIZE_MAX
};

/** @brief convertToFloat() helper function converts the hardware-generated flow vectors to floating point representation (1 flow vector for gridSize).
* gridSize can be queried via function getGridSize().

@param flow Buffer of type CV_16FC2 containing flow vectors generated by calc().
@param floatFlow Buffer of type CV_32FC2, containing flow vectors in floating point representation, each flow vector for 1 pixel per gridSize, in the pitch-linear layout.
*/
CV_WRAP virtual void convertToFloat(InputArray flow, InputOutputArray floatFlow) = 0;

/** @brief Instantiate NVIDIA Optical Flow

@param imageSize Size of input image in pixels.
@param perfPreset Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about presets.
Defaults to NV_OF_PERF_LEVEL_SLOW.
@param outputGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about output grid sizes.
Defaults to NV_OF_OUTPUT_VECTOR_GRID_SIZE_1.
@param hintGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about hint grid sizes.
Defaults to NV_OF_HINT_VECTOR_GRID_SIZE_1.
@param enableTemporalHints Optional parameter. Flag to enable temporal hints. When set to true, the hardware uses the flow vectors
generated in previous call to calc() as internal hints for the current call to calc().
Useful when computing flow vectors between successive video frames. Defaults to false.
@param enableExternalHints Optional Parameter. Flag to enable passing external hints buffer to calc(). Defaults to false.
@param enableCostBuffer Optional Parameter. Flag to enable cost buffer output from calc(). Defaults to false.
@param gpuId Optional parameter to select the GPU ID on which the optical flow should be computed. Useful in multi-GPU systems. Defaults to 0.
@param inputStream Optical flow algorithm may optionally involve cuda preprocessing on the input buffers.
The input cuda stream can be used to pipeline and synchronize the cuda preprocessing tasks with OF HW engine.
If input stream is not set, the execute function will use default stream which is NULL stream;
@param outputStream Optical flow algorithm may optionally involve cuda post processing on the output flow vectors.
The output cuda stream can be used to pipeline and synchronize the cuda post processing tasks with OF HW engine.
If output stream is not set, the execute function will use default stream which is NULL stream;
*/
CV_WRAP static Ptr<NvidiaOpticalFlow_2_0> create(
cv::Size imageSize,
cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL perfPreset
= cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_PERF_LEVEL_SLOW,
cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize
= cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1,
cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize
= cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_HINT_VECTOR_GRID_SIZE_1,
bool enableTemporalHints = false,
bool enableExternalHints = false,
bool enableCostBuffer = false,
int gpuId = 0,
Stream& inputStream = Stream::Null(),
Stream& outputStream = Stream::Null());

/** @brief Instantiate NVIDIA Optical Flow with ROI Feature

@param imageSize Size of input image in pixels.
@param roiData Pointer to ROI data.
@param perfPreset Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about presets.
Defaults to NV_OF_PERF_LEVEL_SLOW.
@param outputGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about output grid sizes.
Defaults to NV_OF_OUTPUT_VECTOR_GRID_SIZE_1.
@param hintGridSize Optional parameter. Refer [NV OF SDK documentation](https://developer.nvidia.com/opticalflow-sdk) for details about hint grid sizes.
Defaults to NV_OF_HINT_VECTOR_GRID_SIZE_1.
@param enableTemporalHints Optional parameter. Flag to enable temporal hints. When set to true, the hardware uses the flow vectors
generated in previous call to calc() as internal hints for the current call to calc().
Useful when computing flow vectors between successive video frames. Defaults to false.
@param enableExternalHints Optional Parameter. Flag to enable passing external hints buffer to calc(). Defaults to false.
@param enableCostBuffer Optional Parameter. Flag to enable cost buffer output from calc(). Defaults to false.
@param gpuId Optional parameter to select the GPU ID on which the optical flow should be computed. Useful in multi-GPU systems. Defaults to 0.
@param inputStream Optical flow algorithm may optionally involve cuda preprocessing on the input buffers.
The input cuda stream can be used to pipeline and synchronize the cuda preprocessing tasks with OF HW engine.
If input stream is not set, the execute function will use default stream which is NULL stream;
@param outputStream Optical flow algorithm may optionally involve cuda post processing on the output flow vectors.
The output cuda stream can be used to pipeline and synchronize the cuda post processing tasks with OF HW engine.
If output stream is not set, the execute function will use default stream which is NULL stream;
*/
CV_WRAP static Ptr<NvidiaOpticalFlow_2_0> create(
cv::Size imageSize,
std::vector<Rect> roiData,
cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL perfPreset
= cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_PERF_LEVEL_SLOW,
cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outputGridSize
= cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1,
cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize
= cv::cuda::NvidiaOpticalFlow_2_0::NV_OF_HINT_VECTOR_GRID_SIZE_1,
bool enableTemporalHints = false,
bool enableExternalHints = false,
bool enableCostBuffer = false,
Expand Down
36 changes: 36 additions & 0 deletions modules/cudaoptflow/misc/python/test/test_nvidiaopticalflow.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,36 @@
import os
import cv2 as cv
import numpy as np

from tests_common import NewOpenCVTests, unittest

class nvidiaopticalflow_test(NewOpenCVTests):
def setUp(self):
super(nvidiaopticalflow_test, self).setUp()
if not cv.cuda.getCudaEnabledDeviceCount():
self.skipTest("No CUDA-capable device is detected")

@unittest.skipIf('OPENCV_TEST_DATA_PATH' not in os.environ,
"OPENCV_TEST_DATA_PATH is not defined")
def test_calc(self):
frame1 = os.environ['OPENCV_TEST_DATA_PATH'] + '/gpu/opticalflow/frame0.png'
frame2 = os.environ['OPENCV_TEST_DATA_PATH'] + '/gpu/opticalflow/frame1.png'

npMat1 = cv.cvtColor(cv.imread(frame1),cv.COLOR_BGR2GRAY)
npMat2 = cv.cvtColor(cv.imread(frame2),cv.COLOR_BGR2GRAY)

cuMat1 = cv.cuda_GpuMat(npMat1)
cuMat2 = cv.cuda_GpuMat(npMat2)
try:
nvof = cv.cuda_NvidiaOpticalFlow_1_0.create(cuMat1.shape[1], cuMat1.shape[0], 5, False, False, False, 0)
flow = nvof.calc(cuMat1, cuMat2, None)
self.assertTrue(flow.shape[1] > 0 and flow.shape[0] > 0)
flowUpSampled = nvof.upSampler(flow[0], cuMat1.shape[1], cuMat1.shape[0], nvof.getGridSize(), None)
nvof.collectGarbage()
except cv.error as e:
if e.code == cv.Error.StsBadFunc or e.code == cv.Error.StsBadArg or e.code == cv.Error.StsNullPtr:
self.skipTest("Algorithm is not supported in the current environment")
self.assertTrue(flowUpSampled.shape[1] > 0 and flowUpSampled.shape[0] > 0)

if __name__ == '__main__':
NewOpenCVTests.bootstrap()
70 changes: 61 additions & 9 deletions modules/cudaoptflow/perf/perf_optflow.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -339,13 +339,8 @@ PERF_TEST_P(ImagePair, NvidiaOpticalFlow_1_0,

const cv::Mat frame1 = readImage(GetParam().second, cv::IMREAD_GRAYSCALE);
ASSERT_FALSE(frame1.empty());

const int width = frame0.size().width;
const int height = frame0.size().height;
const bool enableTemporalHints = false;
const bool enableExternalHints = false;
const bool enableCostBuffer = false;
const int gpuid = 0;
Stream inputStream;
Stream outputStream;

if (PERF_RUN_CUDA())
{
Expand All @@ -355,9 +350,9 @@ PERF_TEST_P(ImagePair, NvidiaOpticalFlow_1_0,
cv::Ptr<cv::cuda::NvidiaOpticalFlow_1_0> d_nvof;
try
{
d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(width, height,
d_nvof = cv::cuda::NvidiaOpticalFlow_1_0::create(frame0.size(),
cv::cuda::NvidiaOpticalFlow_1_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST,
enableTemporalHints, enableExternalHints, enableCostBuffer, gpuid);
false, false, false, 0, inputStream, outputStream);
}
catch (const cv::Exception& e)
{
Expand All @@ -376,6 +371,63 @@ PERF_TEST_P(ImagePair, NvidiaOpticalFlow_1_0,

CUDA_SANITY_CHECK(u, 1e-10);
CUDA_SANITY_CHECK(v, 1e-10);

d_nvof->collectGarbage();
}
}

//////////////////////////////////////////////////////
// NvidiaOpticalFlow_2_0

PERF_TEST_P(ImagePair, NvidiaOpticalFlow_2_0,
Values<pair_string>(make_pair("gpu/opticalflow/frame0.png", "gpu/opticalflow/frame1.png")))
{
declare.time(10);

const cv::Mat frame0 = readImage(GetParam().first, cv::IMREAD_GRAYSCALE);
ASSERT_FALSE(frame0.empty());

const cv::Mat frame1 = readImage(GetParam().second, cv::IMREAD_GRAYSCALE);
ASSERT_FALSE(frame1.empty());

const cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE outGridSize
= cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_OUTPUT_VECTOR_GRID_SIZE::NV_OF_OUTPUT_VECTOR_GRID_SIZE_1;
const cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE hintGridSize
= cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_HINT_VECTOR_GRID_SIZE::NV_OF_HINT_VECTOR_GRID_SIZE_1;
Stream inputStream;
Stream outputStream;

if (PERF_RUN_CUDA())
{
const cv::cuda::GpuMat d_frame0(frame0);
const cv::cuda::GpuMat d_frame1(frame1);
cv::cuda::GpuMat d_flow;
cv::Ptr<cv::cuda::NvidiaOpticalFlow_2_0> d_nvof;
try
{
d_nvof = cv::cuda::NvidiaOpticalFlow_2_0::create(frame0.size(),
cv::cuda::NvidiaOpticalFlow_2_0::NVIDIA_OF_PERF_LEVEL::NV_OF_PERF_LEVEL_FAST, outGridSize, hintGridSize,
false, false, false, 0, inputStream, outputStream);
}
catch (const cv::Exception& e)
{
if (e.code == Error::StsBadFunc || e.code == Error::StsBadArg || e.code == Error::StsNullPtr)
throw SkipTestException("Current configuration is not supported");
throw;
}

TEST_CYCLE() d_nvof->calc(d_frame0, d_frame1, d_flow);

cv::cuda::GpuMat flow[2];
cv::cuda::split(d_flow, flow);

cv::cuda::GpuMat u = flow[0];
cv::cuda::GpuMat v = flow[1];

CUDA_SANITY_CHECK(u, 1e-10);
CUDA_SANITY_CHECK(v, 1e-10);

d_nvof->collectGarbage();
}
}

Expand Down
Loading