Merge pull request #303 from sbokov:improvingStereoSGBM

Author: mshabunin

modules/ximgproc/include/opencv2/ximgproc/disparity_filter.hpp

@@ -39,6 +39,7 @@
 #ifdef __cplusplus
 
 #include <opencv2/core.hpp>
+#include <opencv2/calib3d.hpp>
 
 namespace cv {
 namespace ximgproc {
@@ -63,15 +64,15 @@ class CV_EXPORTS_W DisparityFilter : public Algorithm
 
     @param filtered_disparity_map output disparity map.
 
-    @param ROI region of the disparity map to filter.
-
     @param disparity_map_right optional argument, some implementations might also use the disparity map
     of the right view to compute confidence maps, for instance.
 
+    @param ROI region of the disparity map to filter. Optional, usually it should be set automatically.
+
     @param right_view optional argument, some implementations might also use the right view of the original
     stereo-pair.
      */
-    CV_WRAP virtual void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, Rect ROI, InputArray disparity_map_right = Mat(), InputArray right_view = Mat()) = 0;
+    CV_WRAP virtual void filter(InputArray disparity_map_left, InputArray left_view, OutputArray filtered_disparity_map, InputArray disparity_map_right = Mat(), Rect ROI = Rect(), InputArray right_view = Mat()) = 0;
 };
 
 /** @brief Disparity map filter based on Weighted Least Squares filter (in form of Fast Global Smoother that
@@ -106,8 +107,7 @@ class CV_EXPORTS_W DisparityWLSFilter : public DisparityFilter
     /** @see getLRCthresh */
     CV_WRAP virtual void setLRCthresh(int _LRC_thresh) = 0;
     /** @brief DepthDiscontinuityRadius is a parameter used in confidence computation. It defines the size of
-    low-confidence regions around depth discontinuities. For typical window sizes used in stereo matching the
-    optimal value is around 5.
+    low-confidence regions around depth discontinuities.
      */
     CV_WRAP virtual int getDepthDiscontinuityRadius() = 0;
     /** @see getDepthDiscontinuityRadius */
@@ -117,16 +117,36 @@ class CV_EXPORTS_W DisparityWLSFilter : public DisparityFilter
     correct disparity values with a high degree of confidence).
      */
     CV_WRAP virtual Mat getConfidenceMap() = 0;
-
+    /** @brief Get the ROI used in the last filter call
+     */
+    CV_WRAP virtual Rect getROI() = 0;
 };
 
-/** @brief Factory method, create instance of DisparityWLSFilter and execute the initialization routines.
+/** @brief Convenience factory method that creates an instance of DisparityWLSFilter and sets up all the relevant
+filter parameters automatically based on the matcher instance. Currently supports only StereoBM and StereoSGBM.
+
+@param matcher_left stereo matcher instance that will be used with the filter
+*/
+CV_EXPORTS_W
+Ptr<DisparityWLSFilter> createDisparityWLSFilter(Ptr<StereoMatcher> matcher_left);
+
+/** @brief Convenience method to set up the matcher for computing the right-view disparity map
+that is required in case of filtering with confidence.
+
+@param matcher_left main stereo matcher instance that will be used with the filter
+*/
+CV_EXPORTS_W
+Ptr<StereoMatcher> createRightMatcher(Ptr<StereoMatcher> matcher_left);
+
+/** @brief More generic factory method, create instance of DisparityWLSFilter and execute basic
+initialization routines. When using this method you will need to set-up the ROI, matchers and
+other parameters by yourself.
 
 @param use_confidence filtering with confidence requires two disparity maps (for the left and right views) and is
 approximately two times slower. However, quality is typically significantly better.
 */
 CV_EXPORTS_W
-Ptr<DisparityWLSFilter> createDisparityWLSFilter(bool use_confidence);
+Ptr<DisparityWLSFilter> createDisparityWLSFilterGeneric(bool use_confidence);
 
 //////////////////////////////////////////////////////////////////////////
 //////////////////////////////////////////////////////////////////////////

modules/ximgproc/perf/perf_disparity_wls_filter.cpp

@@ -85,8 +85,8 @@ PERF_TEST_P( DisparityWLSFilterPerfTest, perf, Combine(GuideTypes::all(), SrcTyp
     cv::setNumThreads(cv::getNumberOfCPUs());
     TEST_CYCLE_N(10)
     {
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(use_conf);
-        wls_filter->filter(disp_left,guide,dst,ROI,disp_right);
+        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilterGeneric(use_conf);
+        wls_filter->filter(disp_left,guide,dst,disp_right,ROI);
     }
 
     SANITY_CHECK(dst);

modules/ximgproc/samples/disparity_filtering.cpp

@@ -23,13 +23,13 @@ const String keys =
     "{algorithm      |bm                | stereo matching method (bm or sgbm)                               }"
     "{filter         |wls_conf          | used post-filtering (wls_conf or wls_no_conf)                     }"
     "{no-display     |                  | don't display results                                             }"
-    "{no-downscale   |                  | prevent stereo matching on downscaled views                       }"
+    "{no-downscale   |                  | force stereo matching on full-sized views to improve quality      }"
     "{dst_conf_path  |None              | optional path to save the confidence map used in filtering        }"
     "{vis_mult       |1.0               | coefficient used to scale disparity map visualizations            }"
     "{max_disparity  |160               | parameter of stereo matching                                      }"
-    "{window_size    |19                | parameter of stereo matching                                      }"
+    "{window_size    |-1                | parameter of stereo matching                                      }"
     "{wls_lambda     |8000.0            | parameter of post-filtering                                       }"
-    "{wls_sigma      |1.0               | parameter of post-filtering                                       }"
+    "{wls_sigma      |1.5               | parameter of post-filtering                                       }"
     ;
 
 int main(int argc, char** argv)
@@ -54,17 +54,30 @@ int main(int argc, char** argv)
     bool no_display = parser.has("no-display");
     bool no_downscale = parser.has("no-downscale");
     int max_disp = parser.get<int>("max_disparity");
-    int wsize = parser.get<int>("window_size");
     double lambda = parser.get<double>("wls_lambda");
     double sigma  = parser.get<double>("wls_sigma");
     double vis_mult = parser.get<double>("vis_mult");
 
+    int wsize;
+    if(parser.get<int>("window_size")>=0) //user provided window_size value
+        wsize = parser.get<int>("window_size");
+    else
+    {
+        if(algo=="sgbm")
+            wsize = 3; //default window size for SGBM
+        else if(!no_downscale && algo=="bm" && filter=="wls_conf")
+            wsize = 7; //default window size for BM on downscaled views (downscaling is performed only for wls_conf)
+        else
+            wsize = 15; //default window size for BM on full-sized views
+    }
+
     if (!parser.check())
     {
         parser.printErrors();
         return -1;
     }
 
+    //! [load_views]
     Mat left  = imread(left_im ,IMREAD_COLOR);
     if ( left.empty() )
     {
@@ -78,6 +91,7 @@ int main(int argc, char** argv)
         cout<<"Cannot read image file: "<<right_im;
         return -1;
     }
+    //! [load_views]
 
     bool noGT;
     Mat GT_disp;
@@ -99,6 +113,7 @@ int main(int argc, char** argv)
     Mat conf_map = Mat(left.rows,left.cols,CV_8U);
     conf_map = Scalar(255);
     Rect ROI;
+    Ptr<DisparityWLSFilter> wls_filter;
     double matching_time, filtering_time;
     if(max_disp<=0 || max_disp%16!=0)
     {
@@ -110,56 +125,51 @@ int main(int argc, char** argv)
         cout<<"Incorrect window_size value: it should be positive and odd";
         return -1;
     }
-    if(filter=="wls_conf")
+    if(filter=="wls_conf") // filtering with confidence (significantly better quality than wls_no_conf)
     {
         if(!no_downscale)
         {
-            wsize = wsize/2;
-            if(wsize%2==0) wsize++;
+            // downscale the views to speed-up the matching stage, as we will need to compute both left
+            // and right disparity maps for confidence map computation
+            //! [downscale]
             max_disp/=2;
             if(max_disp%16!=0)
                 max_disp += 16-(max_disp%16);
             resize(left ,left_for_matcher ,Size(),0.5,0.5);
             resize(right,right_for_matcher,Size(),0.5,0.5);
+            //! [downscale]
         }
         else
         {
             left_for_matcher  = left.clone();
             right_for_matcher = right.clone();
         }
 
-
         if(algo=="bm")
         {
-            Ptr<StereoBM> left_matcher  = StereoBM::create(max_disp,wsize);
-            left_matcher->setMinDisparity(0);
-            Ptr<StereoBM> right_matcher = StereoBM::create(max_disp,wsize);
-            right_matcher->setMinDisparity(-max_disp+1);
-            left_matcher->setTextureThreshold(0);
-            left_matcher->setUniquenessRatio(0);
-            right_matcher->setTextureThreshold(0);
-            right_matcher->setUniquenessRatio(0);
-            cvtColor(left_for_matcher,  left_for_matcher, COLOR_BGR2GRAY);
+            //! [matching]
+            Ptr<StereoBM> left_matcher = StereoBM::create(max_disp,wsize);
+            wls_filter = createDisparityWLSFilter(left_matcher);
+            Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
+
+            cvtColor(left_for_matcher,  left_for_matcher,  COLOR_BGR2GRAY);
             cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY);
-            ROI = computeROI(left_for_matcher.size(),left_matcher);
 
             matching_time = (double)getTickCount();
             left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
             right_matcher->compute(right_for_matcher,left_for_matcher, right_disp);
             matching_time = ((double)getTickCount() - matching_time)/getTickFrequency();
+            //! [matching]
         }
         else if(algo=="sgbm")
         {
             Ptr<StereoSGBM> left_matcher  = StereoSGBM::create(0,max_disp,wsize);
-            left_matcher->setMinDisparity(0);
-            Ptr<StereoSGBM> right_matcher = StereoSGBM::create(-max_disp+1,max_disp,wsize);
-            left_matcher->setUniquenessRatio(0);
-            left_matcher->setDisp12MaxDiff(1000000);
-            left_matcher->setSpeckleWindowSize(0);
-            right_matcher->setUniquenessRatio(0);
-            right_matcher->setDisp12MaxDiff(1000000);
-            right_matcher->setSpeckleWindowSize(0);
-            ROI = computeROI(left_for_matcher.size(),left_matcher);
+            left_matcher->setP1(24*wsize*wsize);
+            left_matcher->setP2(96*wsize*wsize);
+            left_matcher->setPreFilterCap(63);
+            left_matcher->setMode(StereoSGBM::MODE_SGBM_3WAY);
+            wls_filter = createDisparityWLSFilter(left_matcher);
+            Ptr<StereoMatcher> right_matcher = createRightMatcher(left_matcher);
 
             matching_time = (double)getTickCount();
             left_matcher-> compute(left_for_matcher, right_for_matcher,left_disp);
@@ -172,14 +182,17 @@ int main(int argc, char** argv)
             return -1;
         }
 
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(true);
+        //! [filtering]
         wls_filter->setLambda(lambda);
         wls_filter->setSigmaColor(sigma);
         filtering_time = (double)getTickCount();
-        wls_filter->filter(left_disp,left,filtered_disp,ROI,right_disp);
+        wls_filter->filter(left_disp,left,filtered_disp,right_disp);
         filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
+        //! [filtering]
         conf_map = wls_filter->getConfidenceMap();
 
+        // Get the ROI that was used in the last filter call:
+        ROI = wls_filter->getROI();
         if(!no_downscale)
         {
             // upscale raw disparity and ROI back for a proper comparison:
@@ -190,6 +203,9 @@ int main(int argc, char** argv)
     }
     else if(filter=="wls_no_conf")
     {
+        /* There is no convenience function for the case of filtering with no confidence, so we
+        will need to set the ROI and matcher parameters manually */
+
         left_for_matcher  = left.clone();
         right_for_matcher = right.clone();
 
@@ -201,6 +217,8 @@ int main(int argc, char** argv)
             cvtColor(left_for_matcher,  left_for_matcher, COLOR_BGR2GRAY);
             cvtColor(right_for_matcher, right_for_matcher, COLOR_BGR2GRAY);
             ROI = computeROI(left_for_matcher.size(),matcher);
+            wls_filter = createDisparityWLSFilterGeneric(false);
+            wls_filter->setDepthDiscontinuityRadius((int)ceil(0.33*wsize));
 
             matching_time = (double)getTickCount();
             matcher->compute(left_for_matcher,right_for_matcher,left_disp);
@@ -212,7 +230,12 @@ int main(int argc, char** argv)
             matcher->setUniquenessRatio(0);
             matcher->setDisp12MaxDiff(1000000);
             matcher->setSpeckleWindowSize(0);
+            matcher->setP1(24*wsize*wsize);
+            matcher->setP2(96*wsize*wsize);
+            matcher->setMode(StereoSGBM::MODE_SGBM_3WAY);
             ROI = computeROI(left_for_matcher.size(),matcher);
+            wls_filter = createDisparityWLSFilterGeneric(false);
+            wls_filter->setDepthDiscontinuityRadius((int)ceil(0.5*wsize));
 
             matching_time = (double)getTickCount();
             matcher->compute(left_for_matcher,right_for_matcher,left_disp);
@@ -224,11 +247,10 @@ int main(int argc, char** argv)
             return -1;
         }
 
-        Ptr<DisparityWLSFilter> wls_filter = createDisparityWLSFilter(false);
         wls_filter->setLambda(lambda);
         wls_filter->setSigmaColor(sigma);
         filtering_time = (double)getTickCount();
-        wls_filter->filter(left_disp,left,filtered_disp,ROI);
+        wls_filter->filter(left_disp,left,filtered_disp,Mat(),ROI);
         filtering_time = ((double)getTickCount() - filtering_time)/getTickFrequency();
     }
     else
@@ -292,6 +314,7 @@ int main(int argc, char** argv)
             imshow("ground-truth disparity", GT_disp_vis);
         }
 
+        //! [visualization]
         Mat raw_disp_vis;
         getDisparityVis(left_disp,raw_disp_vis,vis_mult);
         namedWindow("raw disparity", WINDOW_AUTOSIZE);
@@ -301,6 +324,7 @@ int main(int argc, char** argv)
         namedWindow("filtered disparity", WINDOW_AUTOSIZE);
         imshow("filtered disparity", filtered_disp_vis);
         waitKey();
+        //! [visualization]
     }
 
     return 0;