From 502770f834f25fae415aa0695c0589f406a23ce9 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Mon, 10 Dec 2018 18:46:28 +0000 Subject: [PATCH 01/29] initial commit. --- modules/README.md | 2 + modules/qds/CMakeLists.txt | 3 + modules/qds/README.md | 8 + modules/qds/doc/qds.bib | 36 ++ modules/qds/include/opencv2/qds.hpp | 73 +++ modules/qds/include/opencv2/qds/defaults.hpp | 70 +++ .../include/opencv2/qds/quasiDenseStereo.hpp | 419 +++++++++++++ modules/qds/samples/dense_disparity.cpp | 101 ++++ modules/qds/samples/export_param_file.cpp | 58 ++ modules/qds/src/quasiDenseStereo.cpp | 548 ++++++++++++++++++ .../qds/tutorials/export_param_file.markdown | 26 + .../qds/tutorials/quasi_dense_stereo.markdown | 55 ++ 12 files changed, 1399 insertions(+) create mode 100644 modules/qds/CMakeLists.txt create mode 100644 modules/qds/README.md create mode 100644 modules/qds/doc/qds.bib create mode 100644 modules/qds/include/opencv2/qds.hpp create mode 100644 modules/qds/include/opencv2/qds/defaults.hpp create mode 100644 modules/qds/include/opencv2/qds/quasiDenseStereo.hpp create mode 100644 modules/qds/samples/dense_disparity.cpp create mode 100644 modules/qds/samples/export_param_file.cpp create mode 100644 modules/qds/src/quasiDenseStereo.cpp create mode 100644 modules/qds/tutorials/export_param_file.markdown create mode 100644 modules/qds/tutorials/quasi_dense_stereo.markdown diff --git a/modules/README.md b/modules/README.md index 02db5a8f57b..7bdce47c62c 100644 --- a/modules/README.md +++ b/modules/README.md @@ -46,6 +46,8 @@ $ cmake -D OPENCV_EXTRA_MODULES_PATH=/modules -D BUILD_opencv_ + +/** + * @defgroup qds Quasi Dense Stereo + * This module contains the code to perform quasi dense stereo matching. + * The method initially starts with a sparse 3D reconstruction based on feature matching across a + * stereo image pair and subsequently propagates the structure into neighboring image regions. + * To obtain initial seed correspondences, the algorithm locates Shi and Tomashi features in the + * left image of the stereo pair and then tracks them using pyramidal Lucas-Kanade in the right image. + * To densify the sparse correspondences, the algorithm computes the zero-mean normalized + * cross-correlation (ZNCC) in small patches around every seed pair and uses it as a quality metric + * for each match. In this code, we introduce a custom structure to store the location and ZNCC value + * of correspondences called "Match". Seed Matches are stored in a priority queue sorted according to + * their ZNCC value, allowing for the best quality Match to be readily available. The algorithm pops + * Matches and uses them to extract new matches around them. This is done by considering a small + * neighboring area around each Seed and retrieving correspondences above a certain texture threshold + * that are not previously computed. New matches are stored in the seed priority queue and used as seeds. + * The propagation process ends when no additional matches can be retrieved. + * + * + * @sa This code represents the work presented in @cite Stoyanov2010. + * If this code is useful for your work please cite @cite Stoyanov2010. + * + * Also the original growing scheme idea is described in @cite Lhuillier2000 + * +*/ +#endif // __OPENCV_QDS_H__ diff --git a/modules/qds/include/opencv2/qds/defaults.hpp b/modules/qds/include/opencv2/qds/defaults.hpp new file mode 100644 index 00000000000..855123c01ba --- /dev/null +++ b/modules/qds/include/opencv2/qds/defaults.hpp @@ -0,0 +1,70 @@ +/* +By downloading, copying, installing or using the software you agree to this license. +If you do not agree to this license, do not download, install, +copy or use the software. + + + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) + +Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. +Third party copyrights are property of their respective owners. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are disclaimed. +In no event shall copyright holders or contributors be liable for any direct, +indirect, incidental, special, exemplary, or consequential damages +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. + */ +#ifndef __OPENCV_DEFAULTS_H__ +#define __OPENCV_DEFAULTS_H__ + +// borders around the image +#define BORDER_X 15 +#define BORDER_Y 15 + +#define CORR_WIN_SIZE_X 5 // corr window size +#define CORR_WIN_SIZE_Y 5 + +#define NEIGHBORHOOD_SIZE 5 // neighbors +#define CORR_THRESHOLD 0.5 // corr threshold for seeds +#define TEXTURE_THRESHOLD 200 // texture threshold for seeds +#define DISPARITY_GRADIENT 1 // disparity gradient + + +#define LK_FLOW_TEMPLAETE_SIZE 3 +#define LK_FLOW_PYR_LVL 3 +#define LK_FLOW_TERM_1 3 +#define LK_FLOW_TERM_2 0.003 + +#define GFT_QUALITY_THRESHOLD 0.01 +#define GFT_MIN_SEPERATION_DIST 10 +#define GFT_MAX_NUM_FEATURES 500 + +#define DESPARITY_LVLS 50 + + +#define NO_DISPARITY 0 +#define NO_MATCH cv::Point(0,0) + +#endif //__OPENCV_DEFAULTS_H__ diff --git a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp new file mode 100644 index 00000000000..38ac756788b --- /dev/null +++ b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp @@ -0,0 +1,419 @@ +/* +By downloading, copying, installing or using the software you agree to this license. +If you do not agree to this license, do not download, install, +copy or use the software. + + + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) + +Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. +Third party copyrights are property of their respective owners. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are disclaimed. +In no event shall copyright holders or contributors be liable for any direct, +indirect, incidental, special, exemplary, or consequential damages +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. + */ + + +//__OPENCV_QUASI_DENSE_STEREO_H__ +#ifndef __OPENCV_QUASI_DENSE_STEREO_H__ +#define __OPENCV_QUASI_DENSE_STEREO_H__ + + + +#include +#include //GFT +#include //LK +#include +#include +#include //max +#include +#include +#include +#include +#include + + +#include + +namespace cv +{ +namespace qds +{ +/** \addtogroup qds + * @{ + */ + + +// A basic match structure +struct CV_EXPORTS_W_SIMPLE Match +{ + CV_PROP_RW cv::Point2i p0; + CV_PROP_RW cv::Point2i p1; + CV_PROP_RW float corr; + + CV_WRAP bool operator < (const Match & rhs) const + { + return corr < rhs.corr; + } +}; +struct CV_EXPORTS_W_SIMPLE PropagationParameters +{ + CV_PROP_RW int corrWinSizeX; // similarity window + CV_PROP_RW int corrWinSizeY; + + CV_PROP_RW int borderX; // border to ignore + CV_PROP_RW int borderY; + + //matching + CV_PROP_RW float correlationThreshold; // correlation threshold + CV_PROP_RW float textrureThreshold; // texture threshold + + CV_PROP_RW int neighborhoodSize; // neighborhood size + CV_PROP_RW int disparityGradient; // disparity gradient threshold + + // Parameters for LK flow algorithm + CV_PROP_RW int lkTemplateSize; + CV_PROP_RW int lkPyrLvl; + CV_PROP_RW int lkTermParam1; + CV_PROP_RW float lkTermParam2; + + // Parameters for GFT algorithm. + CV_PROP_RW float gftQualityThres; + CV_PROP_RW int gftMinSeperationDist; + CV_PROP_RW int gftMaxNumFeatures; + +}; + +typedef std::priority_queue, std::less > t_matchPriorityQueue; + + +/** + * @brief Class containing the methods needed for Quasi Dense Stereo computation. + * + */ + +class CV_EXPORTS_W QuasiDenseStereo +{ + + +public: + /** + * @brief constructor + * @param monoImgSize The size of the input images. + * @note Left and right images must be of the same size. + * @param paramFilepath Specifies the location of the file containing the values of all the + * parameters used in this class. + * @note Default value is an an empty string "". In this case the class default parameters, + * found in the defaults.hpp file, are loaded. + */ + QuasiDenseStereo(cv::Size monoImgSize, std::string paramFilepath =""); + + + /** + * @brief destructor + * Method to free all the memory allocated by matrices and vectors in this class. + */ + virtual ~QuasiDenseStereo(); + + + /** + * @brief Load a file containing the configuration parameters of the class. + * @param[in] filepath The location of the .YAML file containing the configuration parameters. + * @note default value is an empty string in which case the default + * parameters, specified in the qds/defaults.h header-file, are loaded. + * @retval 1: If the path is not empty and the program loaded the parameters successfully. + * @retval 0: If the path is empty and the program loaded default parameters. + * @retval -1: If the file location is not valid or the program could not open the file and + * loaded default parameters from defaults.hpp. + * @note The method is automatically called in the constructor and configures the class. + * @note Loading different parameters will have an effect on the output. This is useful for tuning + * in case of video processing. + * @sa loadParameters + */ + CV_WRAP int loadParameters(std::string filepath=""); + + + /** + * @brief Save a file containing all the configuration parameters the class is currently set to. + * @param[in] filepath The location to store the parameters file. + * @note Calling this method with no arguments will result in storing class parameters to a file + * names "qds_parameters.yaml" in the root project folder. + * @note This method can be used to generate a template file for tuning the class. + * @sa loadParameters + */ + CV_WRAP int saveParameters(std::string filepath="./qds_parameters.yaml"); + + + /** + * @brief Get The sparse corresponding points. + * @param[out] sMatches A vector containing all sparse correspondences. + * @note The method clears the sMatches vector. + * @note The returned Match elements inside the sMatches vector, do not use corr member. + */ + CV_WRAP void getSparseMatches(std::vector &sMatches); + + + /** + * @brief Get The dense corresponding points. + * @param[out] dMatches A vector containing all dense matches. + * @note The method clears the dMatches vector. + * @note The returned Match elements inside the sMatches vector, do not use corr member. + */ + CV_WRAP void getDenseMatches(std::vector &dMatches); + + + + /** + * @brief Main process of the algorithm. This method computes the sparse seeds and then densifies them. + * + * Initially input images are converted to gray-scale and then the sparseMatching method + * is called to obtain the sparse stereo. Finally quasiDenseMatching is called to densify the corresponding + * points. + * @param[in] imgLeft The left Channel of a stereo image pair. + * @param[in] imgRight The right Channel of a stereo image pair. + * @note If input images are in color, the method assumes that are BGR and converts them to grayscale. + * @sa sparseMatching + * @sa quasiDenseMatching + */ + CV_WRAP void process(const cv::Mat &imgLeft ,const cv::Mat &imgRight); + + + /** + * @brief Specify pixel coordinates in the left image and get its corresponding location in the right image. + * @param[in] x The x pixel coordinate in the left image channel. + * @param[in] y The y pixel coordinate in the left image channel. + * @retval cv::Point(x, y) The location of the corresponding pixel in the right image. + * @retval cv::Point(0, 0) (NO_MATCH) if no match is found in the right image for the specified pixel location in the left image. + * @note This method should be always called after process, otherwise the matches will not be correct. + */ + CV_WRAP cv::Point2f getMatch(const int x, const int y); + + + /** + * @brief Compute and return the disparity map based on the correspondences found in the "process" method. + * @param[in] disparityLvls The level of detail in output disparity image. + * @note Default level is 50 + * @return cv::Mat containing a the disparity image in grayscale. + * @sa computeDisparity + * @sa quantizeDisparity + */ + CV_WRAP cv::Mat getDisparity(uint8_t disparityLvls=50); + + + + PropagationParameters Param; + +protected: + /** + * @brief Computes sparse stereo. The output is stores in refMap and mthMap. + * + * This method used the "goodFeaturesToTrack" function of OpenCV to extracts salient points + * in the left image. Feature locations are used as inputs in the "calcOpticalFlowPyrLK" + * function of OpenCV along with the left and right images. The optical flow algorithm estimates + * tracks the locations of the features in the right image. The two set of locations constitute + * the sparse set of matches. These are then used as seeds in the intensification stage of the algorithm. + * @param[in] imgLeft The left Channel of a stereo image. + * @param[in] imgRight The right Channel of a stereo image. + * @param[out] featuresLeft (vector of points) The location of the features in the left image. + * @param[out] featuresRight (vector of points) The location of the features in the right image. + * @note featuresLeft and featuresRight must have the same length and corresponding features + * must be indexed the same way in both vectors. + */ + CV_WRAP virtual void sparseMatching(const cv::Mat &imgLeft ,const cv::Mat &imgRight, + std::vector< cv::Point2f > &featuresLeft, + std::vector< cv::Point2f > &featuresRight); + + + /** + * @brief Based on the seeds computed in sparse stereo, this method calculates the semi dense set of correspondences. + * + * The method initially discards low quality matches based on their zero-normalized cross correlation (zncc) value. + * This is done by calling the "extractSparseSeeds" method. Remaining high quality Matches are stored in a t_matchPriorityQueue + * sorted according to their zncc value. The priority queue allows for new matches to be added while keeping track + * of the best Match. The algorithm then process the queue iteratively. + * In every iteration a Match is popped from the queue. The algorithm then tries to find candidate + * matches by matching every point in a small patch around the left Match feature, with a point + * within a same sized patch around the corresponding right feature. For each candidate point match, + * the zncc is computed and if it surpasses a threshold, the candidate pair is stored in a temporary + * priority queue. After this process completed the candidate matches are popped from the Local + * priority queue and if a match is not registered in refMap, it means that is the best match for + * this point. The algorithm registers this point in refMap and also push it to the Seed queue. + * if a candidate match is already registered, it means that is not the best and the algorithm + * discards it. + * + * @note This method does not have input arguments, but uses the "leftFeatures" and "rightFeatures" vectors. + * Also there is no output since the method used refMap and mtcMap to store the results. + * @param[in] featuresLeft The location of the features in the left image. + * @param[in] featuresRight The location of the features in the right image. + */ + CV_WRAP void quasiDenseMatching(const std::vector< cv::Point2f > &featuresLeft, + const std::vector< cv::Point2f > &featuresRight); + + + /** + * @brief Compute the disparity map based on the Euclidean distance of corresponding points. + * @param[in] matchMap A matrix of points, the same size as the left channel. Each cell of this + * matrix stores the location of the corresponding point in the right image. + * @param[out] dispMat The disparity map. + * @sa quantizeDisparity + * @sa getDisparity + */ + CV_WRAP void computeDisparity(const cv::Mat_ &matchMap, + cv::Mat_ &dispMat); + + + /** + * @brief Disparity map normalization for display purposes. If needed specify the quantization level as input argument. + * @param[in] dispMat The disparity Map. + * @param[in] lvls The quantization level of the output disparity map. + * @return Disparity image. + * @note Stores the output in the disparityImage class variable. + * @sa computeDisparity + * @sa quantiseDisparity + */ + CV_WRAP cv::Mat quantiseDisparity(const cv::Mat_ &dispMat, const int lvls); + + + /** + * @brief Compute the Zero-mean Normalized Cross-correlation. + * + * Compare a patch in the left image, centered in point p0 with a patch in the right image, centered in point p1. + * Patches are defined by wy, wx and the patch size is (2*wx+1) by (2*wy+1). + * @param [in] p0 The central point of the patch in the left image. + * @param [in] p1 The central point of the patch in the right image. + * @param [in] wx The distance from the center of the patch to the border in the x direction. + * @param [in] wy The distance from the center of the patch to the border in the y direction. + * @return The value of the the zero-mean normalized cross correlation. + * @note Default value for wx, wy is 1. in this case the patch is 3x3. + */ + CV_WRAP float iZNCC_c1(const cv::Point2i p0, const cv::Point2i p1, const int wx=1, const int wy=1); + + + /** + * @brief Compute the sum of values and the sum of squared values of a patch with dimensions + * 2*xWindow+1 by 2*yWindow+1 and centered in point p, using the integral image and integral image of squared pixel values. + * @param[in] p The center of the patch we want to calculate the sum and sum of squared values. + * @param[in] s The integral image + * @param[in] ss The integral image of squared values. + * @param[out] sum The sum of pixels inside the patch. + * @param[out] ssum The sum of squared values inside the patch. + * @param [in] xWindow The distance from the central pixel of the patch to the border in x direction. + * @param [in] yWindow The distance from the central pixel of the patch to the border in y direction. + * @note Default value for xWindow, yWindow is 1. in this case the patch is 3x3. + * @note integral images are very useful to sum values of patches in constant time independent of their + * size. For more information refer to the cv::Integral function OpenCV page. + */ + CV_WRAP void patchSumSum2(const cv::Point2i p, const cv::Mat &sum, const cv::Mat &ssum, + float &s, float &ss, const int xWindow=1, const int yWindow=1); + + + /** + * @brief Create a priority queue containing sparse Matches + * + * This method computes the zncc for each Match extracted in "sparseMatching". If the zncc is over + * the correlation threshold then the Match is inserted in the output priority queue. + * @param[in] featuresLeft The feature locations in the left image. + * @param[in] featuresRight The features locations in the right image. + * @param[out] leftMap A matrix of points, of the same size as the left image. Each cell of this + * matrix stores the location of the corresponding point in the right image. + * @param[out] rightMap A matrix of points, the same size as the right image. Each cell of this + * matrix stores the location of the corresponding point in the left image. + * @return Priority queue containing sparse matches. + */ + CV_WRAP t_matchPriorityQueue extractSparseSeeds(const std::vector< cv::Point2f > &featuresLeft, + const std::vector< cv::Point2f > &featuresRight, + cv::Mat_ &leftMap, + cv::Mat_ &rightMap); + + + /** + * @brief Check if a match is close to the boarder of an image. + * @param[in] m The match containing points in both image. + * @param[in] bx The offset of the image edge that defines the border in x direction. + * @param[in] by The offset of the image edge that defines the border in y direction. + * @param[in] w The width of the image. + * @param[in] h The height of the image. + * @retval true If the feature is in the border of the image. + * @retval false If the feature is not in the border of image. + */ + CV_WRAP bool CheckBorder(Match m, int bx, int by, int w, int h); + + + /** + * @brief Compare two matches based on their zncc correlation values. + * @param[in] a First Match. + * @param[in] b Second Match. + * @retval true If b is equal or grater Match than a. + * @retval false If b is less Match than a. + */ + CV_PROP_RW CV_WRAP bool MatchCompare(const Match a, const Match b); + + + /** + * @brief Build a texture descriptor + * @param[in] img The image we need to compute the descriptor for. + * @param[out] descriptor The texture descriptor of the image. + */ + CV_PROP_RW CV_WRAP void buildTextureDescriptor(cv::Mat &img,cv::Mat &descriptor); + + + // Variables used at sparse feature extraction. + // Container for left images' features, extracted with GFT algorithm. + CV_PROP_RW std::vector< cv::Point2f > leftFeatures; + // Container for right images' features, matching is done with LK flow algorithm. + CV_PROP_RW std::vector< cv::Point2f > rightFeatures; + + // Width and height of a single image. + CV_PROP_RW int width; + CV_PROP_RW int height; + CV_PROP_RW int dMatchesLen; + // Containers to store input images. + CV_PROP_RW cv::Mat grayLeft; + CV_PROP_RW cv::Mat grayRight; + // Containers to store the locations of each points pair. + CV_PROP_RW cv::Mat_ refMap; + CV_PROP_RW cv::Mat_ mtcMap; + CV_PROP_RW cv::Mat_ sum0; + CV_PROP_RW cv::Mat_ sum1; + CV_PROP_RW cv::Mat_ ssum0; + CV_PROP_RW cv::Mat_ ssum1; + // Container to store the disparity un-normalized + CV_PROP_RW cv::Mat_ disparity; + // Container to store the disparity image. + CV_PROP_RW cv::Mat_ disparityImg; + // Containers to store textures descriptors. + CV_PROP_RW cv::Mat_ textureDescLeft; + CV_PROP_RW cv::Mat_ textureDescRight; + +}; + +} //namespace cv +} //namespace qds + +/** @}*/ + +#endif // __OPENCV_QUASI_DENSE_STEREO_H__ diff --git a/modules/qds/samples/dense_disparity.cpp b/modules/qds/samples/dense_disparity.cpp new file mode 100644 index 00000000000..6f471e81bc7 --- /dev/null +++ b/modules/qds/samples/dense_disparity.cpp @@ -0,0 +1,101 @@ +/* +By downloading, copying, installing or using the software you agree to this license. +If you do not agree to this license, do not download, install, +copy or use the software. + + + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) + +Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. +Third party copyrights are property of their respective owners. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are disclaimed. +In no event shall copyright holders or contributors be liable for any direct, +indirect, incidental, special, exemplary, or consequential damages + + +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, + +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. + */ + + +#include +#include +#include +#include + + + + +using namespace cv; +using namespace std; + + +int main() +{ + + Mat rightImg, leftImg; + + //Read video meta-data to determine the correct frame size for initialization. + leftImg = imread("./imgLeft.png", IMREAD_COLOR); + rightImg = imread("./imgRight.png", IMREAD_COLOR); + cv::Size frameSize = leftImg.size(); + // Initialize qds and start process. + qds::QuasiDenseStereo stereo(frameSize); + + int displvl = 80; // Number of disparity levels + cv::Mat disp; + + // Compute dense stereo. + stereo.process(leftImg, rightImg); + + // Compute disparity between left and right channel of current frame. + disp = stereo.getDisparity(displvl); + + vector matches; + stereo.getDenseMatches(matches); + + // Create three windows and show images. + cv::namedWindow("right channel"); + cv::namedWindow("left channel"); + cv::namedWindow("disparity map"); + cv::imshow("disparity map", disp); + cv::imshow("left channel", leftImg); + cv::imshow("right channel", rightImg); + + std::ofstream dense("./dense.txt", std::ios::out); + + for (uint i=0; i< matches.size(); i++) + { + dense << matches[i].p0 << matches[i].p1 << endl; + } + dense.close(); + + + + cv::waitKey(0); + + return 0; +} diff --git a/modules/qds/samples/export_param_file.cpp b/modules/qds/samples/export_param_file.cpp new file mode 100644 index 00000000000..e1b62f5a881 --- /dev/null +++ b/modules/qds/samples/export_param_file.cpp @@ -0,0 +1,58 @@ +/* +By downloading, copying, installing or using the software you agree to this license. +If you do not agree to this license, do not download, install, +copy or use the software. + + + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) + +Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. +Third party copyrights are property of their respective owners. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are disclaimed. +In no event shall copyright holders or contributors be liable for any direct, +indirect, incidental, special, exemplary, or consequential damages + + +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, + +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. + */ + +#include +#include + +using namespace cv; +using namespace std; + +int main(int argc, char* argv[]) +{ + std::string parameterFileLocation = ""; + if (argc > 1) + parameterFileLocation = argv[1]; + + qds::QuasiDenseStereo(cv::Size(5,5)).saveParameters(parameterFileLocation); + + return 0; +} diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/qds/src/quasiDenseStereo.cpp new file mode 100644 index 00000000000..010cfc7ce39 --- /dev/null +++ b/modules/qds/src/quasiDenseStereo.cpp @@ -0,0 +1,548 @@ +/* +By downloading, copying, installing or using the software you agree to this license. +If you do not agree to this license, do not download, install, +copy or use the software. + + + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) + +Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. +Third party copyrights are property of their respective owners. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are disclaimed. +In no event shall copyright holders or contributors be liable for any direct, +indirect, incidental, special, exemplary, or consequential damages +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. + */ + +#include "opencv2/qds/quasiDenseStereo.hpp" + +namespace cv { +namespace qds { + +QuasiDenseStereo::QuasiDenseStereo(cv::Size monoImgSize, std::string paramFilepath) +{ + loadParameters(paramFilepath); + width = monoImgSize.width; + height = monoImgSize.height; + refMap = cv::Mat_(monoImgSize); + mtcMap = cv::Mat_(monoImgSize); + + cv::Size integralSize = cv::Size(monoImgSize.width+1, monoImgSize.height+1); + sum0 = cv::Mat_(integralSize); + sum1 = cv::Mat_(integralSize); + ssum0 = cv::Mat_(integralSize); + ssum1 = cv::Mat_(integralSize); + // the disparity image. + disparity = cv::Mat_(monoImgSize); + disparityImg = cv::Mat_(monoImgSize); + // texture images. + textureDescLeft = cv::Mat_ (monoImgSize); + textureDescRight = cv::Mat_ (monoImgSize); +} + +QuasiDenseStereo::~QuasiDenseStereo() +{ + + rightFeatures.clear(); + leftFeatures.clear(); + + refMap.release(); + mtcMap.release(); + + sum0.release(); + sum1.release(); + ssum0.release(); + ssum1.release(); + // the disparity image. + disparity.release(); + disparityImg.release(); + // texture images. + textureDescLeft.release(); + textureDescRight.release(); +} + + +int QuasiDenseStereo::loadParameters(std::string filepath) +{ + cv::FileStorage fs; + //if user specified a pathfile, try to use it. + if (!filepath.empty()) + { + fs.open(filepath, cv::FileStorage::READ); + } + // If the file opened, read the parameters. + if (fs.isOpened()) + { + fs["borderX"] >> Param.borderX; + fs["borderY"] >> Param.borderY; + fs["corrWinSizeX"] >> Param.corrWinSizeX; + fs["corrWinSizeY"] >> Param.corrWinSizeY; + fs["correlationThreshold"] >> Param.correlationThreshold; + fs["textrureThreshold"] >> Param.textrureThreshold; + + fs["neighborhoodSize"] >> Param.neighborhoodSize; + fs["disparityGradient"] >> Param.disparityGradient; + + fs["lkTemplateSize"] >> Param.lkTemplateSize; + fs["lkPyrLvl"] >> Param.lkPyrLvl; + fs["lkTermParam1"] >> Param.lkTermParam1; + fs["lkTermParam2"] >> Param.lkTermParam2; + + fs["gftQualityThres"] >> Param.gftQualityThres; + fs["gftMinSeperationDist"] >> Param.gftMinSeperationDist; + fs["gftMaxNumFeatures"] >> Param.gftMaxNumFeatures; + fs.release(); + return 1; + } + // If the filepath was incorrect or non existent, load the defaults. + Param.borderX = BORDER_X; + Param.borderY = BORDER_Y; + Param.corrWinSizeX = CORR_WIN_SIZE_X; + Param.corrWinSizeY = CORR_WIN_SIZE_Y; + Param.correlationThreshold = CORR_THRESHOLD; + Param.textrureThreshold = TEXTURE_THRESHOLD; + + Param.neighborhoodSize = NEIGHBORHOOD_SIZE; + Param.disparityGradient = DISPARITY_GRADIENT; + + Param.lkTemplateSize = LK_FLOW_TEMPLAETE_SIZE; + Param.lkPyrLvl = LK_FLOW_PYR_LVL; + Param.lkTermParam1 = LK_FLOW_TERM_1; + Param.lkTermParam2 = LK_FLOW_TERM_2; + + Param.gftQualityThres = GFT_QUALITY_THRESHOLD; + Param.gftMinSeperationDist = GFT_MIN_SEPERATION_DIST; + Param.gftMaxNumFeatures = GFT_MAX_NUM_FEATURES; + // Return 0 if there was no filepath provides. + // Return -1 if there was a problem opening the filepath provided. + if(filepath.empty()) + { + return 0; + } + return -1; +} + +int QuasiDenseStereo::saveParameters(std::string filepath) +{ + cv::FileStorage fs(filepath, cv::FileStorage::WRITE); + if (fs.isOpened()) + { + fs << "borderX" << Param.borderX; + fs << "borderY" << Param.borderY; + fs << "corrWinSizeX" << Param.corrWinSizeX; + fs << "corrWinSizeY" << Param.corrWinSizeY; + fs << "correlationThreshold" << Param.correlationThreshold; + fs << "textrureThreshold" << Param.textrureThreshold; + + fs << "neighborhoodSize" << Param.neighborhoodSize; + fs << "disparityGradient" << Param.disparityGradient; + + fs << "lkTemplateSize" << Param.lkTemplateSize; + fs << "lkPyrLvl" << Param.lkPyrLvl; + fs << "lkTermParam1" << Param.lkTermParam1; + fs << "lkTermParam2" << Param.lkTermParam2; + + fs << "gftQualityThres" << Param.gftQualityThres; + fs << "gftMinSeperationDist" << Param.gftMinSeperationDist; + fs << "gftMaxNumFeatures" << Param.gftMaxNumFeatures; + fs.release(); + } + return -1; +} + +void QuasiDenseStereo::getSparseMatches(std::vector &sMatches) +{ + Match tmpMatch; + sMatches.clear(); + sMatches.resize(leftFeatures.size()); + for (uint i=0; i &dMatches) +{ + Match tmpMatch; + dMatches.clear(); +// dMatches.resize(dMatchesLen); + for (int row=0; row(row, col); + if (tmpMatch.p1 == NO_MATCH) + { + continue; + } + dMatches.push_back(tmpMatch); + } + } +} + +void QuasiDenseStereo::process(const cv::Mat &imgLeft , const cv::Mat &imgRight) +{ + if (imgLeft.channels()>1) + { + cv::cvtColor(imgLeft, grayLeft, cv::COLOR_BGR2GRAY); + cv::cvtColor(imgRight, grayRight, cv::COLOR_BGR2GRAY); + } + else + { + grayLeft = imgLeft.clone(); + grayRight = imgRight.clone(); + } + sparseMatching(grayLeft, grayRight, leftFeatures, rightFeatures); + quasiDenseMatching(leftFeatures, rightFeatures); +} + + +cv::Point2f QuasiDenseStereo::getMatch(const int x, const int y) +{ + return refMap.at(y, x); +} + + +void QuasiDenseStereo::sparseMatching(const cv::Mat &imgLeft ,const cv::Mat &imgRight, + std::vector< cv::Point2f > &featuresLeft, + std::vector< cv::Point2f > &featuresRight) +{ + std::vector< uchar > featureStatus; + std::vector< float > error; + featuresLeft.clear(); + featuresRight.clear(); + + cv::goodFeaturesToTrack(imgLeft, featuresLeft, Param.gftMaxNumFeatures, + Param.gftQualityThres, Param.gftMinSeperationDist); + + cv::Size templateSize(Param.lkTemplateSize,Param.lkTemplateSize); + cv::TermCriteria termination(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, + Param.lkTermParam1, Param.lkTermParam2); + cv::calcOpticalFlowPyrLK(imgLeft, imgRight, featuresLeft, featuresRight, + featureStatus, error, + templateSize, Param.lkPyrLvl, termination); + //discard bad features. + for(size_t i=0; i(p0.y+row, p0.x+col) * + (float)grayRight.at(p1.y+row, p1.x+col); + } + } + zncc = (zncc-wa*m0*m1)/(s0*s1); + return zncc; +} + +void QuasiDenseStereo::patchSumSum2(const cv::Point2i p, const cv::Mat &sum, const cv::Mat &ssum, + float &s, float &ss, const int xWindow, const int yWindow) +{ + cv::Point2i otl(p.x-xWindow, p.y-yWindow); + //outer top right + cv::Point2i otr(p.x+xWindow+1, p.y-yWindow); + //outer bottom left + cv::Point2i obl(p.x-xWindow, p.y+yWindow+1); + //outer bottom right + cv::Point2i obr(p.x+xWindow+1, p.y+yWindow+1); + + // sum and squared sum for right window + s = sum.at(otl) - sum.at(otr) + -sum.at(obl) + sum.at(obr); + + ss = ssum.at(otl) - ssum.at(otr) + -ssum.at(obl) + ssum.at(obr); +} + + + +void QuasiDenseStereo::buildTextureDescriptor(cv::Mat &src,cv::Mat &descriptor) +{ + + float a, b, c, d; + + //for each pixel in the input image. The boundaries are this way not to raise errors when computing a,b,c,d? + + uint8_t center, top, bottom, right, left; + //reset descriptors + + // traverse every pixel. + for(int row=1; row(row,col); + top = src.at(row-1,col); + bottom = src.at(row+1,col); + left = src.at(row,col-1); + right = src.at(row,col+1); + + a = (float)abs(center - top); + b = (float)abs(center - bottom); + c = (float)abs(center - left); + d = (float)abs(center - right); + //choose the biggest of them. + int val = std::max(a, std::max(b, std::max(c, d))); + descriptor.at(row, col) = val; + } + } +} + +bool QuasiDenseStereo::CheckBorder(Match m, int bx, int by, int w, int h) +{ + if(m.p0.xw-bx || m.p0.yh-by || + m.p1.xw-bx || m.p1.yh-by) + { + return false; + } + + return true; +} + +bool QuasiDenseStereo::MatchCompare(const Match a, const Match b) +{ + if(a.corr<=b.corr)return true; + return false; +} + +t_matchPriorityQueue QuasiDenseStereo::extractSparseSeeds(const std::vector< cv::Point2f > &featuesLeft, + const std::vector< cv::Point2f > &featuresRight, + cv::Mat_ &leftMap, + cv::Mat_ &rightMap) +{ + t_matchPriorityQueue seeds; + for(uint i=0; i < featuesLeft.size(); i++) + { + // Calculate correlation and store match in Seeds. + Match m; + m.p0 = cv::Point2i(featuesLeft[i]); + m.p1 = cv::Point2i(featuresRight[i]); + m.corr = 0; + + // Check if too close to boundary. + if(!CheckBorder(m,Param.borderX,Param.borderY, width, height)) + continue; + + m.corr = iZNCC_c1(m.p0, m.p1, Param.corrWinSizeX, Param.corrWinSizeY); + // Can we add it to the list + if( m.corr > Param.correlationThreshold ) + { + seeds.push(m); + leftMap.at(m.p0.y, m.p0.x) = m.p1; + rightMap.at(m.p1.y, m.p1.x) = m.p0; + } + } + return seeds; +} + +void QuasiDenseStereo::quasiDenseMatching(const std::vector< cv::Point2f > &featuresLeft, + const std::vector< cv::Point2f > &featuresRight) +{ + refMap = cv::Mat_(cv::Size(width, height), cv::Point2i(0, 0)); + mtcMap = cv::Point2i(0, 0); + + // build texture homogeneity reference maps. + buildTextureDescriptor(grayLeft, textureDescLeft); + buildTextureDescriptor(grayRight, textureDescRight); + + // generate the intergal images for fast variable window correlation calculations + cv::integral(grayLeft, sum0, ssum0); + cv::integral(grayRight, sum1, ssum1); + + // Seed priority queue. The algorithm wants to pop the best seed available in order to densify the mess. // Evs mess ??? the sparse set maybe better ?? !!!!!!! + t_matchPriorityQueue seeds = extractSparseSeeds(featuresLeft, featuresRight, + refMap, mtcMap); + + + // Do the propagation part + while(!seeds.empty()) + { + t_matchPriorityQueue Local; + + // Get the best seed at the moment + Match m = seeds.top(); + seeds.pop(); + + // Ignore the border + if(!CheckBorder(m, Param.borderX, Param.borderY, width, height)) + continue; + + // For all neighbours of the seed in image 1 + //the neighborghoud is defined with Param.N*2 dimentrion + for(int y=-Param.neighborhoodSize;y<=Param.neighborhoodSize;y++) + { + for(int x=-Param.neighborhoodSize;x<=Param.neighborhoodSize;x++) + { + CvPoint p0 = cvPoint(m.p0.x+x,m.p0.y+y); + + // Check if its unique in ref + if(refMap.at(p0.y,p0.x) != NO_MATCH) + continue; + + // Check the texture descriptor for a boundary + if(textureDescLeft.at(p0.y, p0.x) > Param.textrureThreshold) + continue; + + // For all candidate matches. + for(int wy=-Param.disparityGradient; wy<=Param.disparityGradient; wy++) + { + for(int wx=-Param.disparityGradient; wx<=Param.disparityGradient; wx++) + { + cv::Point p1 = cv::Point(m.p1.x+x+wx,m.p1.y+y+wy); + + // Check if its unique in ref + if(mtcMap.at(p1.y, p1.x) != NO_MATCH) + continue; + + // Check the texture descriptor for a boundary + if(textureDescRight.at(p1.y, p1.x) > Param.textrureThreshold) + continue; + + // Calculate ZNCC and store local match. + float corr = iZNCC_c1(p0,p1,Param.corrWinSizeX,Param.corrWinSizeY); + + // push back if this is valid match + if( corr > Param.correlationThreshold ) + { + Match nm; + nm.p0 = p0; + nm.p1 = p1; + nm.corr = corr; + Local.push(nm); + } + } + } + } + } + + // Get seeds from the local + while( !Local.empty() ) + { + Match lm = Local.top(); + Local.pop(); + // Check if its unique in both ref and dst. + if(refMap.at(lm.p0.y, lm.p0.x) != NO_MATCH) + continue; + if(mtcMap.at(lm.p1.y, lm.p1.x) != NO_MATCH) + continue; + + + // Unique match + refMap.at(lm.p0.y, lm.p0.x) = lm.p1; + mtcMap.at(lm.p1.y, lm.p1.x) = lm.p0; + // Add to the seed list + seeds.push(lm); + } + } +} + +void QuasiDenseStereo::computeDisparity(const cv::Mat_ &matchMap, + cv::Mat_ &dispMat) +{ + for(int row=0; row< height; row++) + { + for(int col=0; col(tmpPoint) == NO_MATCH) + { + dispMat.at(tmpPoint) = 200; + continue; + } + //if a match is found, compute the difference in location of the match and current pixel. + float dx = col-matchMap.at(tmpPoint).x; + float dy = row-matchMap.at(tmpPoint).y; + //calculate disparity of current pixel. + dispMat.at(tmpPoint) = sqrt(float(dx*dx+dy*dy)); + } + } +} + +cv::Mat QuasiDenseStereo::quantiseDisparity(const cv::Mat_ &dispMat, const int lvls) +{ + float tmpPixelVal ; + double min, max; +// cv::minMaxLoc(disparity, &min, &max); + min = 0; + max = lvls; + for(int row=0; row(row, col); + tmpPixelVal = 255. - 255.0*(tmpPixelVal-min)/(max-min); + + disparityImg.at(row, col) = (uint8_t) tmpPixelVal; + } + } + return disparityImg; +} + +cv::Mat QuasiDenseStereo::getDisparity(uint8_t disparityLvls) +{ + computeDisparity(refMap, disparity); + return quantiseDisparity(disparity, disparityLvls); +} + +} +} diff --git a/modules/qds/tutorials/export_param_file.markdown b/modules/qds/tutorials/export_param_file.markdown new file mode 100644 index 00000000000..8ceb35ad8b3 --- /dev/null +++ b/modules/qds/tutorials/export_param_file.markdown @@ -0,0 +1,26 @@ +Exporting a template parameter file {#export_param_file} +================== + +Goal +---- + +In this tutorial you will learn how to + +- create a simple parameter file template. + +----------- +[Source Code](../samples/export_param_file.cpp) + +## Explanation: + +The class supports loading configuration parameters from a .yaml file using the method `loadParameters()`. +This is very useful for fine-tuning the class' parameters on the fly. To extract a template of this parameter +file you just run 2 lines of code. +``` +parameterFileLocation = "./parameters.yaml" +qds::QuasiDenseStereo(cv::Size(5,5)).saveParameters(parameterFileLocation); + +``` +We make an instance of a `QuasiDenseStereo` object. Not specifying the second argument of the constructor, +makes the object to load default parameters from [default.h](../include/opencv2/qds/defaults.hpp). +By calling the method `saveParameters()`, we store the template file to the location specified by `parameterFileLocation` diff --git a/modules/qds/tutorials/quasi_dense_stereo.markdown b/modules/qds/tutorials/quasi_dense_stereo.markdown new file mode 100644 index 00000000000..a964b694069 --- /dev/null +++ b/modules/qds/tutorials/quasi_dense_stereo.markdown @@ -0,0 +1,55 @@ +Quasi dense Stereo {#quasi_dense_stereo} +================== + +Goal +---- + +In this tutorial you will learn how to + +- Configure a QuasiDenseStero object +- Compute dense Stereo correspondences. + +----------- +[Source Code](../samples/dense_disparity.cpp) + +## Explanation: + +The program loads a stereo image pair. + + +After importing the images. +``` +Mat rightImg, leftImg; +leftImg = imread("./imgLeft.png", IMREAD_COLOR); +rightImg = imread("./imgRight.png", IMREAD_COLOR); +``` +We need to know the frame size of a single image, in order to create an instance of a `QuasiDesnseStereo` object. +``` +cv::Size frameSize = leftImg.size(); +qds::QuasiDenseStereo stereo(frameSize); +``` +Because we didn't specify the second argument in the constructor, the `QuasiDesnseStereo` object will +load default parameters from this [headerfile](../include/opencv2/qds/defaults.hpp). + +We can then pass the imported stereo images in the process method like this +``` +stereo.process(leftChannel, rightChannel); +``` +The process method contains most of the functionality of the class and does two main things. +- Computes a sparse stereo based in "Good Features to Track" and "pyramidal Lucas-Kanade" flow algorithm +- Based on those sparse stereo points, densifies the stereo correspondences using Quasi Dense Stereo method. + +After the execution of `process()` we can display the disparity Image of the stereo. +``` +int displvl = 80; +Mat disp; +disp = stereo.getDisparity(displvl); +cv::namedWindow("disparity map"); +cv::imshow("disparity map", disp); +``` + +At this point we can also extract all the corresponding points using `getDenseMatches()` method. +``` +vector matches; +stereo.getDenseMatches(matches); +``` From e3301f44ef932b00e29e1ac2c336ad09238de5f6 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 12 Dec 2018 12:18:54 +0000 Subject: [PATCH 02/29] Remove license header. --- modules/qds/samples/dense_disparity.cpp | 43 ----------------------- modules/qds/samples/export_param_file.cpp | 42 ---------------------- 2 files changed, 85 deletions(-) diff --git a/modules/qds/samples/dense_disparity.cpp b/modules/qds/samples/dense_disparity.cpp index 6f471e81bc7..f1705f0aa38 100644 --- a/modules/qds/samples/dense_disparity.cpp +++ b/modules/qds/samples/dense_disparity.cpp @@ -1,46 +1,3 @@ -/* -By downloading, copying, installing or using the software you agree to this license. -If you do not agree to this license, do not download, install, -copy or use the software. - - - License Agreement - For Open Source Computer Vision Library - (3-clause BSD License) - -Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. -Third party copyrights are property of their respective owners. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the names of the copyright holders nor the names of the contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -This software is provided by the copyright holders and contributors "as is" and -any express or implied warranties, including, but not limited to, the implied -warranties of merchantability and fitness for a particular purpose are disclaimed. -In no event shall copyright holders or contributors be liable for any direct, -indirect, incidental, special, exemplary, or consequential damages - - -(including, but not limited to, procurement of substitute goods or services; -loss of use, data, or profits; or business interruption) however caused -and on any theory of liability, whether in contract, strict liability, - -or tort (including negligence or otherwise) arising in any way out of -the use of this software, even if advised of the possibility of such damage. - */ - - #include #include #include diff --git a/modules/qds/samples/export_param_file.cpp b/modules/qds/samples/export_param_file.cpp index e1b62f5a881..56f550f287b 100644 --- a/modules/qds/samples/export_param_file.cpp +++ b/modules/qds/samples/export_param_file.cpp @@ -1,45 +1,3 @@ -/* -By downloading, copying, installing or using the software you agree to this license. -If you do not agree to this license, do not download, install, -copy or use the software. - - - License Agreement - For Open Source Computer Vision Library - (3-clause BSD License) - -Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. -Third party copyrights are property of their respective owners. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the names of the copyright holders nor the names of the contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -This software is provided by the copyright holders and contributors "as is" and -any express or implied warranties, including, but not limited to, the implied -warranties of merchantability and fitness for a particular purpose are disclaimed. -In no event shall copyright holders or contributors be liable for any direct, -indirect, incidental, special, exemplary, or consequential damages - - -(including, but not limited to, procurement of substitute goods or services; -loss of use, data, or profits; or business interruption) however caused -and on any theory of liability, whether in contract, strict liability, - -or tort (including negligence or otherwise) arising in any way out of -the use of this software, even if advised of the possibility of such damage. - */ - #include #include From 529f8eb33fd50b6eb265e6301063da2ae56ddd04 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 12 Dec 2018 12:26:19 +0000 Subject: [PATCH 03/29] Fix python wrap flags --- modules/qds/include/opencv2/qds/quasiDenseStereo.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp index 38ac756788b..60b52d0fbe4 100644 --- a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp +++ b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp @@ -75,7 +75,7 @@ struct CV_EXPORTS_W_SIMPLE Match CV_PROP_RW cv::Point2i p1; CV_PROP_RW float corr; - CV_WRAP bool operator < (const Match & rhs) const + CV_WRAP_AS(less) bool operator < (const Match & rhs) const { return corr < rhs.corr; } @@ -370,7 +370,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @retval true If b is equal or grater Match than a. * @retval false If b is less Match than a. */ - CV_PROP_RW CV_WRAP bool MatchCompare(const Match a, const Match b); + CV_WRAP bool MatchCompare(const Match a, const Match b); /** @@ -378,7 +378,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @param[in] img The image we need to compute the descriptor for. * @param[out] descriptor The texture descriptor of the image. */ - CV_PROP_RW CV_WRAP void buildTextureDescriptor(cv::Mat &img,cv::Mat &descriptor); + CV_WRAP void buildTextureDescriptor(cv::Mat &img,cv::Mat &descriptor); // Variables used at sparse feature extraction. From 5952b83fea5e1f84c81b9a68097e7006377ccbea Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 12 Dec 2018 12:55:02 +0000 Subject: [PATCH 04/29] Change std::string to cv::String, in function declarations, to resolve compilation issues. --- modules/qds/include/opencv2/qds/quasiDenseStereo.hpp | 6 +++--- modules/qds/src/quasiDenseStereo.cpp | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp index 60b52d0fbe4..4598dc5ae5d 100644 --- a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp +++ b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp @@ -130,7 +130,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note Default value is an an empty string "". In this case the class default parameters, * found in the defaults.hpp file, are loaded. */ - QuasiDenseStereo(cv::Size monoImgSize, std::string paramFilepath =""); + QuasiDenseStereo(cv::Size monoImgSize, cv::String paramFilepath =""); /** @@ -154,7 +154,7 @@ class CV_EXPORTS_W QuasiDenseStereo * in case of video processing. * @sa loadParameters */ - CV_WRAP int loadParameters(std::string filepath=""); + CV_WRAP int loadParameters(cv::String filepath=""); /** @@ -165,7 +165,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note This method can be used to generate a template file for tuning the class. * @sa loadParameters */ - CV_WRAP int saveParameters(std::string filepath="./qds_parameters.yaml"); + CV_WRAP int saveParameters(cv::String filepath="./qds_parameters.yaml"); /** diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/qds/src/quasiDenseStereo.cpp index 010cfc7ce39..9d6193bb1bb 100644 --- a/modules/qds/src/quasiDenseStereo.cpp +++ b/modules/qds/src/quasiDenseStereo.cpp @@ -42,7 +42,7 @@ the use of this software, even if advised of the possibility of such damage. namespace cv { namespace qds { -QuasiDenseStereo::QuasiDenseStereo(cv::Size monoImgSize, std::string paramFilepath) +QuasiDenseStereo::QuasiDenseStereo(cv::Size monoImgSize, cv::String paramFilepath) { loadParameters(paramFilepath); width = monoImgSize.width; @@ -85,7 +85,7 @@ QuasiDenseStereo::~QuasiDenseStereo() } -int QuasiDenseStereo::loadParameters(std::string filepath) +int QuasiDenseStereo::loadParameters(cv::String filepath) { cv::FileStorage fs; //if user specified a pathfile, try to use it. @@ -145,7 +145,7 @@ int QuasiDenseStereo::loadParameters(std::string filepath) return -1; } -int QuasiDenseStereo::saveParameters(std::string filepath) +int QuasiDenseStereo::saveParameters(cv::String filepath) { cv::FileStorage fs(filepath, cv::FileStorage::WRITE); if (fs.isOpened()) From 6bb29e0536c3dc26c3529290078061203e048bc3 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 12 Dec 2018 15:05:28 +0000 Subject: [PATCH 05/29] Add python wrapper extending header --- .../misc/python/pyopencv_quasiDenseStereo.hpp | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp diff --git a/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp b/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp new file mode 100644 index 00000000000..6c3956e24cf --- /dev/null +++ b/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp @@ -0,0 +1,18 @@ +#ifdef HAVE_OPENCV_QDS +#include "opencv2/core/saturate.hpp" + +template<> struct pyopencvVecConverter +{ + static bool to(PyObject* obj, std::vector& value, const ArgInfo info) + { + return pyopencv_to_generic_vec(obj, value, info); + } + + static PyObject* from(const std::vector& value) + { + return pyopencv_from_generic_vec(value); + } +}; +typedef std::vector vector_qds_Match; + +#endif From 928a47f15ef5924221cdd101e1f2d013cd89409f Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 12 Dec 2018 15:08:45 +0000 Subject: [PATCH 06/29] Fix python wrapper conflicts --- modules/qds/include/opencv2/qds/quasiDenseStereo.hpp | 4 ++-- modules/qds/src/quasiDenseStereo.cpp | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp index 4598dc5ae5d..8fe3a85ef3d 100644 --- a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp +++ b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp @@ -174,7 +174,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note The method clears the sMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - CV_WRAP void getSparseMatches(std::vector &sMatches); + CV_WRAP void getSparseMatches(std::vector &sMatches); /** @@ -183,7 +183,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note The method clears the dMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - CV_WRAP void getDenseMatches(std::vector &dMatches); + CV_WRAP void getDenseMatches(std::vector &dMatches); diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/qds/src/quasiDenseStereo.cpp index 9d6193bb1bb..c1a4d768eb1 100644 --- a/modules/qds/src/quasiDenseStereo.cpp +++ b/modules/qds/src/quasiDenseStereo.cpp @@ -173,7 +173,7 @@ int QuasiDenseStereo::saveParameters(cv::String filepath) return -1; } -void QuasiDenseStereo::getSparseMatches(std::vector &sMatches) +void QuasiDenseStereo::getSparseMatches(std::vector &sMatches) { Match tmpMatch; sMatches.clear(); @@ -185,7 +185,7 @@ void QuasiDenseStereo::getSparseMatches(std::vector &sMatches) sMatches.push_back(tmpMatch); } } -void QuasiDenseStereo::getDenseMatches(std::vector &dMatches) +void QuasiDenseStereo::getDenseMatches(std::vector &dMatches) { Match tmpMatch; dMatches.clear(); From 17dde34532d45e7d0ba91bf9fec0c53e8cc04b06 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 12 Dec 2018 17:00:12 +0000 Subject: [PATCH 07/29] Fix implicit type conversions --- modules/qds/include/opencv2/qds/defaults.hpp | 6 +++--- modules/qds/samples/dense_disparity.cpp | 2 +- modules/qds/src/quasiDenseStereo.cpp | 14 ++++++-------- 3 files changed, 10 insertions(+), 12 deletions(-) diff --git a/modules/qds/include/opencv2/qds/defaults.hpp b/modules/qds/include/opencv2/qds/defaults.hpp index 855123c01ba..9c95ecd867a 100644 --- a/modules/qds/include/opencv2/qds/defaults.hpp +++ b/modules/qds/include/opencv2/qds/defaults.hpp @@ -47,7 +47,7 @@ the use of this software, even if advised of the possibility of such damage. #define CORR_WIN_SIZE_Y 5 #define NEIGHBORHOOD_SIZE 5 // neighbors -#define CORR_THRESHOLD 0.5 // corr threshold for seeds +#define CORR_THRESHOLD 0.5f // corr threshold for seeds #define TEXTURE_THRESHOLD 200 // texture threshold for seeds #define DISPARITY_GRADIENT 1 // disparity gradient @@ -55,9 +55,9 @@ the use of this software, even if advised of the possibility of such damage. #define LK_FLOW_TEMPLAETE_SIZE 3 #define LK_FLOW_PYR_LVL 3 #define LK_FLOW_TERM_1 3 -#define LK_FLOW_TERM_2 0.003 +#define LK_FLOW_TERM_2 0.003f -#define GFT_QUALITY_THRESHOLD 0.01 +#define GFT_QUALITY_THRESHOLD 0.01f #define GFT_MIN_SEPERATION_DIST 10 #define GFT_MAX_NUM_FEATURES 500 diff --git a/modules/qds/samples/dense_disparity.cpp b/modules/qds/samples/dense_disparity.cpp index f1705f0aa38..d8a86cfcc38 100644 --- a/modules/qds/samples/dense_disparity.cpp +++ b/modules/qds/samples/dense_disparity.cpp @@ -22,7 +22,7 @@ int main() // Initialize qds and start process. qds::QuasiDenseStereo stereo(frameSize); - int displvl = 80; // Number of disparity levels + uint8_t displvl = 80; // Number of disparity levels cv::Mat disp; // Compute dense stereo. diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/qds/src/quasiDenseStereo.cpp index c1a4d768eb1..321390f2f78 100644 --- a/modules/qds/src/quasiDenseStereo.cpp +++ b/modules/qds/src/quasiDenseStereo.cpp @@ -305,11 +305,9 @@ void QuasiDenseStereo::patchSumSum2(const cv::Point2i p, const cv::Mat &sum, con cv::Point2i obr(p.x+xWindow+1, p.y+yWindow+1); // sum and squared sum for right window - s = sum.at(otl) - sum.at(otr) - -sum.at(obl) + sum.at(obr); + s = (float)(sum.at(otl) - sum.at(otr) - sum.at(obl) + sum.at(obr)); - ss = ssum.at(otl) - ssum.at(otr) - -ssum.at(obl) + ssum.at(obr); + ss = (float)(ssum.at(otl) - ssum.at(otr) - ssum.at(obl) + ssum.at(obr)); } @@ -341,7 +339,7 @@ void QuasiDenseStereo::buildTextureDescriptor(cv::Mat &src,cv::Mat &descriptor) c = (float)abs(center - left); d = (float)abs(center - right); //choose the biggest of them. - int val = std::max(a, std::max(b, std::max(c, d))); + int val = (int) std::max(a, std::max(b, std::max(c, d))); descriptor.at(row, col) = val; } } @@ -510,8 +508,8 @@ void QuasiDenseStereo::computeDisparity(const cv::Mat_ &matchMap, continue; } //if a match is found, compute the difference in location of the match and current pixel. - float dx = col-matchMap.at(tmpPoint).x; - float dy = row-matchMap.at(tmpPoint).y; + int dx = col-matchMap.at(tmpPoint).x; + int dy = row-matchMap.at(tmpPoint).y; //calculate disparity of current pixel. dispMat.at(tmpPoint) = sqrt(float(dx*dx+dy*dy)); } @@ -530,7 +528,7 @@ cv::Mat QuasiDenseStereo::quantiseDisparity(const cv::Mat_ &dispMat, cons for(int col=0; col(row, col); - tmpPixelVal = 255. - 255.0*(tmpPixelVal-min)/(max-min); + tmpPixelVal = (float) (255. - 255.0*(tmpPixelVal-min)/(max-min)); disparityImg.at(row, col) = (uint8_t) tmpPixelVal; } From 3f90782efe264ffa5d519c985960e33f019865c7 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Thu, 13 Dec 2018 17:08:27 +0000 Subject: [PATCH 08/29] Change C API types and enums to C++. --- modules/qds/src/quasiDenseStereo.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/qds/src/quasiDenseStereo.cpp index 321390f2f78..6c60745487c 100644 --- a/modules/qds/src/quasiDenseStereo.cpp +++ b/modules/qds/src/quasiDenseStereo.cpp @@ -241,8 +241,8 @@ void QuasiDenseStereo::sparseMatching(const cv::Mat &imgLeft ,const cv::Mat &img Param.gftQualityThres, Param.gftMinSeperationDist); cv::Size templateSize(Param.lkTemplateSize,Param.lkTemplateSize); - cv::TermCriteria termination(CV_TERMCRIT_ITER|CV_TERMCRIT_EPS, - Param.lkTermParam1, Param.lkTermParam2); + cv::TermCriteria termination(cv::TermCriteria::MAX_ITER | cv::TermCriteria::EPS, + Param.lkTermParam1, Param.lkTermParam2); cv::calcOpticalFlowPyrLK(imgLeft, imgRight, featuresLeft, featuresRight, featureStatus, error, templateSize, Param.lkPyrLvl, termination); @@ -430,7 +430,7 @@ void QuasiDenseStereo::quasiDenseMatching(const std::vector< cv::Point2f > &feat { for(int x=-Param.neighborhoodSize;x<=Param.neighborhoodSize;x++) { - CvPoint p0 = cvPoint(m.p0.x+x,m.p0.y+y); + cv::Point2i p0 = cv::Point2i(m.p0.x+x,m.p0.y+y); // Check if its unique in ref if(refMap.at(p0.y,p0.x) != NO_MATCH) From 88094637f30f1d8f759daad77ae778dbc87846ba Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Thu, 13 Dec 2018 17:29:19 +0000 Subject: [PATCH 09/29] Remove redundant included headers and move wanted headers to src/precomp.hpp --- .../include/opencv2/qds/quasiDenseStereo.hpp | 16 ++----- modules/qds/samples/dense_disparity.cpp | 2 +- modules/qds/src/precomp.hpp | 48 +++++++++++++++++++ modules/qds/src/quasiDenseStereo.cpp | 2 +- 4 files changed, 53 insertions(+), 15 deletions(-) create mode 100644 modules/qds/src/precomp.hpp diff --git a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp index 8fe3a85ef3d..c7b21f0e130 100644 --- a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp +++ b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp @@ -45,19 +45,9 @@ the use of this software, even if advised of the possibility of such damage. #include -#include //GFT -#include //LK -#include -#include -#include //max #include -#include -#include -#include -#include -#include namespace cv { @@ -75,9 +65,9 @@ struct CV_EXPORTS_W_SIMPLE Match CV_PROP_RW cv::Point2i p1; CV_PROP_RW float corr; - CV_WRAP_AS(less) bool operator < (const Match & rhs) const + CV_WRAP_AS(less) bool operator < (const Match & rhs) const//fixme may be used uninitialized in this function { - return corr < rhs.corr; + return this->corr < rhs.corr; } }; struct CV_EXPORTS_W_SIMPLE PropagationParameters @@ -225,7 +215,7 @@ class CV_EXPORTS_W QuasiDenseStereo - PropagationParameters Param; + CV_PROP_RW PropagationParameters Param; protected: /** diff --git a/modules/qds/samples/dense_disparity.cpp b/modules/qds/samples/dense_disparity.cpp index d8a86cfcc38..ff1e0419d0f 100644 --- a/modules/qds/samples/dense_disparity.cpp +++ b/modules/qds/samples/dense_disparity.cpp @@ -1,6 +1,6 @@ #include #include -#include +#include #include diff --git a/modules/qds/src/precomp.hpp b/modules/qds/src/precomp.hpp new file mode 100644 index 00000000000..851081a8b9f --- /dev/null +++ b/modules/qds/src/precomp.hpp @@ -0,0 +1,48 @@ +/* +By downloading, copying, installing or using the software you agree to this license. +If you do not agree to this license, do not download, install, +copy or use the software. + + + License Agreement + For Open Source Computer Vision Library + (3-clause BSD License) + +Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. +Third party copyrights are property of their respective owners. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + + * Neither the names of the copyright holders nor the names of the contributors + may be used to endorse or promote products derived from this software + without specific prior written permission. + +This software is provided by the copyright holders and contributors "as is" and +any express or implied warranties, including, but not limited to, the implied +warranties of merchantability and fitness for a particular purpose are disclaimed. +In no event shall copyright holders or contributors be liable for any direct, +indirect, incidental, special, exemplary, or consequential damages +(including, but not limited to, procurement of substitute goods or services; +loss of use, data, or profits; or business interruption) however caused +and on any theory of liability, whether in contract, strict liability, +or tort (including negligence or otherwise) arising in any way out of +the use of this software, even if advised of the possibility of such damage. + */ + +#ifndef __OPENCV_PRECOMP_H__ +#define __OPENCV_PRECOMP_H__ + +#include +#include +#include +#include + +#endif diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/qds/src/quasiDenseStereo.cpp index 6c60745487c..b0685192afa 100644 --- a/modules/qds/src/quasiDenseStereo.cpp +++ b/modules/qds/src/quasiDenseStereo.cpp @@ -37,7 +37,7 @@ or tort (including negligence or otherwise) arising in any way out of the use of this software, even if advised of the possibility of such damage. */ -#include "opencv2/qds/quasiDenseStereo.hpp" +#include "precomp.hpp" namespace cv { namespace qds { From 791e8239daf6056fb319acd4f503fbecc443c18b Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Thu, 13 Dec 2018 17:31:27 +0000 Subject: [PATCH 10/29] Remove saturate header --- modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp b/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp index 6c3956e24cf..30adc97082c 100644 --- a/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp +++ b/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp @@ -1,5 +1,4 @@ #ifdef HAVE_OPENCV_QDS -#include "opencv2/core/saturate.hpp" template<> struct pyopencvVecConverter { From 80627544f9529958f14340745be2252dc39395db Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Thu, 13 Dec 2018 18:16:38 +0000 Subject: [PATCH 11/29] Remove unnecessary python wrapping flags --- .../include/opencv2/qds/quasiDenseStereo.hpp | 56 +++++++++---------- 1 file changed, 28 insertions(+), 28 deletions(-) diff --git a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp index c7b21f0e130..5bde6a93645 100644 --- a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp +++ b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp @@ -65,7 +65,7 @@ struct CV_EXPORTS_W_SIMPLE Match CV_PROP_RW cv::Point2i p1; CV_PROP_RW float corr; - CV_WRAP_AS(less) bool operator < (const Match & rhs) const//fixme may be used uninitialized in this function + bool operator < (const Match & rhs) const//fixme may be used uninitialized in this function { return this->corr < rhs.corr; } @@ -233,7 +233,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note featuresLeft and featuresRight must have the same length and corresponding features * must be indexed the same way in both vectors. */ - CV_WRAP virtual void sparseMatching(const cv::Mat &imgLeft ,const cv::Mat &imgRight, + virtual void sparseMatching(const cv::Mat &imgLeft ,const cv::Mat &imgRight, std::vector< cv::Point2f > &featuresLeft, std::vector< cv::Point2f > &featuresRight); @@ -260,7 +260,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @param[in] featuresLeft The location of the features in the left image. * @param[in] featuresRight The location of the features in the right image. */ - CV_WRAP void quasiDenseMatching(const std::vector< cv::Point2f > &featuresLeft, + void quasiDenseMatching(const std::vector< cv::Point2f > &featuresLeft, const std::vector< cv::Point2f > &featuresRight); @@ -272,7 +272,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @sa quantizeDisparity * @sa getDisparity */ - CV_WRAP void computeDisparity(const cv::Mat_ &matchMap, + void computeDisparity(const cv::Mat_ &matchMap, cv::Mat_ &dispMat); @@ -285,7 +285,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @sa computeDisparity * @sa quantiseDisparity */ - CV_WRAP cv::Mat quantiseDisparity(const cv::Mat_ &dispMat, const int lvls); + cv::Mat quantiseDisparity(const cv::Mat_ &dispMat, const int lvls); /** @@ -300,7 +300,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @return The value of the the zero-mean normalized cross correlation. * @note Default value for wx, wy is 1. in this case the patch is 3x3. */ - CV_WRAP float iZNCC_c1(const cv::Point2i p0, const cv::Point2i p1, const int wx=1, const int wy=1); + float iZNCC_c1(const cv::Point2i p0, const cv::Point2i p1, const int wx=1, const int wy=1); /** @@ -317,7 +317,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note integral images are very useful to sum values of patches in constant time independent of their * size. For more information refer to the cv::Integral function OpenCV page. */ - CV_WRAP void patchSumSum2(const cv::Point2i p, const cv::Mat &sum, const cv::Mat &ssum, + void patchSumSum2(const cv::Point2i p, const cv::Mat &sum, const cv::Mat &ssum, float &s, float &ss, const int xWindow=1, const int yWindow=1); @@ -334,7 +334,7 @@ class CV_EXPORTS_W QuasiDenseStereo * matrix stores the location of the corresponding point in the left image. * @return Priority queue containing sparse matches. */ - CV_WRAP t_matchPriorityQueue extractSparseSeeds(const std::vector< cv::Point2f > &featuresLeft, + t_matchPriorityQueue extractSparseSeeds(const std::vector< cv::Point2f > &featuresLeft, const std::vector< cv::Point2f > &featuresRight, cv::Mat_ &leftMap, cv::Mat_ &rightMap); @@ -350,7 +350,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @retval true If the feature is in the border of the image. * @retval false If the feature is not in the border of image. */ - CV_WRAP bool CheckBorder(Match m, int bx, int by, int w, int h); + bool CheckBorder(Match m, int bx, int by, int w, int h); /** @@ -360,7 +360,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @retval true If b is equal or grater Match than a. * @retval false If b is less Match than a. */ - CV_WRAP bool MatchCompare(const Match a, const Match b); + bool MatchCompare(const Match a, const Match b); /** @@ -368,36 +368,36 @@ class CV_EXPORTS_W QuasiDenseStereo * @param[in] img The image we need to compute the descriptor for. * @param[out] descriptor The texture descriptor of the image. */ - CV_WRAP void buildTextureDescriptor(cv::Mat &img,cv::Mat &descriptor); + void buildTextureDescriptor(cv::Mat &img,cv::Mat &descriptor); // Variables used at sparse feature extraction. // Container for left images' features, extracted with GFT algorithm. - CV_PROP_RW std::vector< cv::Point2f > leftFeatures; + std::vector< cv::Point2f > leftFeatures; // Container for right images' features, matching is done with LK flow algorithm. - CV_PROP_RW std::vector< cv::Point2f > rightFeatures; + std::vector< cv::Point2f > rightFeatures; // Width and height of a single image. - CV_PROP_RW int width; - CV_PROP_RW int height; - CV_PROP_RW int dMatchesLen; + int width; + int height; + int dMatchesLen; // Containers to store input images. - CV_PROP_RW cv::Mat grayLeft; - CV_PROP_RW cv::Mat grayRight; + cv::Mat grayLeft; + cv::Mat grayRight; // Containers to store the locations of each points pair. - CV_PROP_RW cv::Mat_ refMap; - CV_PROP_RW cv::Mat_ mtcMap; - CV_PROP_RW cv::Mat_ sum0; - CV_PROP_RW cv::Mat_ sum1; - CV_PROP_RW cv::Mat_ ssum0; - CV_PROP_RW cv::Mat_ ssum1; + cv::Mat_ refMap; + cv::Mat_ mtcMap; + cv::Mat_ sum0; + cv::Mat_ sum1; + cv::Mat_ ssum0; + cv::Mat_ ssum1; // Container to store the disparity un-normalized - CV_PROP_RW cv::Mat_ disparity; + cv::Mat_ disparity; // Container to store the disparity image. - CV_PROP_RW cv::Mat_ disparityImg; + cv::Mat_ disparityImg; // Containers to store textures descriptors. - CV_PROP_RW cv::Mat_ textureDescLeft; - CV_PROP_RW cv::Mat_ textureDescRight; + cv::Mat_ textureDescLeft; + cv::Mat_ textureDescRight; }; From 0a4f040bb7650bb7083c3c4747ca7fec5f44bd57 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Fri, 14 Dec 2018 12:10:58 +0000 Subject: [PATCH 12/29] Removed defaults parameter header --- modules/qds/include/opencv2/qds/defaults.hpp | 70 -------------------- modules/qds/src/precomp.hpp | 1 - modules/qds/src/quasiDenseStereo.cpp | 41 ++++++------ 3 files changed, 22 insertions(+), 90 deletions(-) delete mode 100644 modules/qds/include/opencv2/qds/defaults.hpp diff --git a/modules/qds/include/opencv2/qds/defaults.hpp b/modules/qds/include/opencv2/qds/defaults.hpp deleted file mode 100644 index 9c95ecd867a..00000000000 --- a/modules/qds/include/opencv2/qds/defaults.hpp +++ /dev/null @@ -1,70 +0,0 @@ -/* -By downloading, copying, installing or using the software you agree to this license. -If you do not agree to this license, do not download, install, -copy or use the software. - - - License Agreement - For Open Source Computer Vision Library - (3-clause BSD License) - -Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. -Third party copyrights are property of their respective owners. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the names of the copyright holders nor the names of the contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -This software is provided by the copyright holders and contributors "as is" and -any express or implied warranties, including, but not limited to, the implied -warranties of merchantability and fitness for a particular purpose are disclaimed. -In no event shall copyright holders or contributors be liable for any direct, -indirect, incidental, special, exemplary, or consequential damages -(including, but not limited to, procurement of substitute goods or services; -loss of use, data, or profits; or business interruption) however caused -and on any theory of liability, whether in contract, strict liability, -or tort (including negligence or otherwise) arising in any way out of -the use of this software, even if advised of the possibility of such damage. - */ -#ifndef __OPENCV_DEFAULTS_H__ -#define __OPENCV_DEFAULTS_H__ - -// borders around the image -#define BORDER_X 15 -#define BORDER_Y 15 - -#define CORR_WIN_SIZE_X 5 // corr window size -#define CORR_WIN_SIZE_Y 5 - -#define NEIGHBORHOOD_SIZE 5 // neighbors -#define CORR_THRESHOLD 0.5f // corr threshold for seeds -#define TEXTURE_THRESHOLD 200 // texture threshold for seeds -#define DISPARITY_GRADIENT 1 // disparity gradient - - -#define LK_FLOW_TEMPLAETE_SIZE 3 -#define LK_FLOW_PYR_LVL 3 -#define LK_FLOW_TERM_1 3 -#define LK_FLOW_TERM_2 0.003f - -#define GFT_QUALITY_THRESHOLD 0.01f -#define GFT_MIN_SEPERATION_DIST 10 -#define GFT_MAX_NUM_FEATURES 500 - -#define DESPARITY_LVLS 50 - - -#define NO_DISPARITY 0 -#define NO_MATCH cv::Point(0,0) - -#endif //__OPENCV_DEFAULTS_H__ diff --git a/modules/qds/src/precomp.hpp b/modules/qds/src/precomp.hpp index 851081a8b9f..5d355a2998e 100644 --- a/modules/qds/src/precomp.hpp +++ b/modules/qds/src/precomp.hpp @@ -43,6 +43,5 @@ the use of this software, even if advised of the possibility of such damage. #include #include #include -#include #endif diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/qds/src/quasiDenseStereo.cpp index b0685192afa..157a79c6637 100644 --- a/modules/qds/src/quasiDenseStereo.cpp +++ b/modules/qds/src/quasiDenseStereo.cpp @@ -42,6 +42,8 @@ the use of this software, even if advised of the possibility of such damage. namespace cv { namespace qds { +#define NO_MATCH cv::Point(0,0) + QuasiDenseStereo::QuasiDenseStereo(cv::Size monoImgSize, cv::String paramFilepath) { loadParameters(paramFilepath); @@ -117,25 +119,26 @@ int QuasiDenseStereo::loadParameters(cv::String filepath) fs.release(); return 1; } - // If the filepath was incorrect or non existent, load the defaults. - Param.borderX = BORDER_X; - Param.borderY = BORDER_Y; - Param.corrWinSizeX = CORR_WIN_SIZE_X; - Param.corrWinSizeY = CORR_WIN_SIZE_Y; - Param.correlationThreshold = CORR_THRESHOLD; - Param.textrureThreshold = TEXTURE_THRESHOLD; - - Param.neighborhoodSize = NEIGHBORHOOD_SIZE; - Param.disparityGradient = DISPARITY_GRADIENT; - - Param.lkTemplateSize = LK_FLOW_TEMPLAETE_SIZE; - Param.lkPyrLvl = LK_FLOW_PYR_LVL; - Param.lkTermParam1 = LK_FLOW_TERM_1; - Param.lkTermParam2 = LK_FLOW_TERM_2; - - Param.gftQualityThres = GFT_QUALITY_THRESHOLD; - Param.gftMinSeperationDist = GFT_MIN_SEPERATION_DIST; - Param.gftMaxNumFeatures = GFT_MAX_NUM_FEATURES; + // If the filepath was incorrect or non existent, load default parameters. + Param.borderX = 15; + Param.borderY = 15; + // corr window size + Param.corrWinSizeX = 5; + Param.corrWinSizeY = 5; + Param.correlationThreshold = (float)0.5; + Param.textrureThreshold = 200; + + Param.neighborhoodSize = 5; + Param.disparityGradient = 1; + + Param.lkTemplateSize = 3; + Param.lkPyrLvl = 3; + Param.lkTermParam1 = 3; + Param.lkTermParam2 = (float)0.003; + + Param.gftQualityThres = (float)0.01; + Param.gftMinSeperationDist = 10; + Param.gftMaxNumFeatures = 500; // Return 0 if there was no filepath provides. // Return -1 if there was a problem opening the filepath provided. if(filepath.empty()) From 19e42b01facbc8fe03a65b3d6542b8ed80beeb91 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Mon, 17 Dec 2018 19:19:49 +0000 Subject: [PATCH 13/29] Split declaration and implementation of the class using Pimpl. --- .../include/opencv2/qds/quasiDenseStereo.hpp | 215 +--- modules/qds/src/quasiDenseStereo.cpp | 994 ++++++++++-------- 2 files changed, 589 insertions(+), 620 deletions(-) diff --git a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp index 5bde6a93645..12a5fb485d2 100644 --- a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp +++ b/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp @@ -108,26 +108,12 @@ typedef std::priority_queue, std::less > t_matc class CV_EXPORTS_W QuasiDenseStereo { - - public: - /** - * @brief constructor - * @param monoImgSize The size of the input images. - * @note Left and right images must be of the same size. - * @param paramFilepath Specifies the location of the file containing the values of all the - * parameters used in this class. - * @note Default value is an an empty string "". In this case the class default parameters, - * found in the defaults.hpp file, are loaded. - */ - QuasiDenseStereo(cv::Size monoImgSize, cv::String paramFilepath =""); - - /** * @brief destructor * Method to free all the memory allocated by matrices and vectors in this class. */ - virtual ~QuasiDenseStereo(); + virtual ~QuasiDenseStereo() = 0; /** @@ -144,7 +130,7 @@ class CV_EXPORTS_W QuasiDenseStereo * in case of video processing. * @sa loadParameters */ - CV_WRAP int loadParameters(cv::String filepath=""); + CV_WRAP virtual int loadParameters(cv::String filepath="") = 0; /** @@ -155,7 +141,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note This method can be used to generate a template file for tuning the class. * @sa loadParameters */ - CV_WRAP int saveParameters(cv::String filepath="./qds_parameters.yaml"); + CV_WRAP virtual int saveParameters(cv::String filepath="./qds_parameters.yaml") = 0; /** @@ -164,7 +150,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note The method clears the sMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - CV_WRAP void getSparseMatches(std::vector &sMatches); + CV_WRAP virtual void getSparseMatches(std::vector &sMatches) = 0; /** @@ -173,7 +159,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note The method clears the dMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - CV_WRAP void getDenseMatches(std::vector &dMatches); + CV_WRAP virtual void getDenseMatches(std::vector &dMatches) = 0; @@ -189,7 +175,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @sa sparseMatching * @sa quasiDenseMatching */ - CV_WRAP void process(const cv::Mat &imgLeft ,const cv::Mat &imgRight); + CV_WRAP virtual void process(const cv::Mat &imgLeft ,const cv::Mat &imgRight) = 0; /** @@ -200,7 +186,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @retval cv::Point(0, 0) (NO_MATCH) if no match is found in the right image for the specified pixel location in the left image. * @note This method should be always called after process, otherwise the matches will not be correct. */ - CV_WRAP cv::Point2f getMatch(const int x, const int y); + CV_WRAP virtual cv::Point2f getMatch(const int x, const int y) = 0; /** @@ -211,194 +197,13 @@ class CV_EXPORTS_W QuasiDenseStereo * @sa computeDisparity * @sa quantizeDisparity */ - CV_WRAP cv::Mat getDisparity(uint8_t disparityLvls=50); - - - - CV_PROP_RW PropagationParameters Param; - -protected: - /** - * @brief Computes sparse stereo. The output is stores in refMap and mthMap. - * - * This method used the "goodFeaturesToTrack" function of OpenCV to extracts salient points - * in the left image. Feature locations are used as inputs in the "calcOpticalFlowPyrLK" - * function of OpenCV along with the left and right images. The optical flow algorithm estimates - * tracks the locations of the features in the right image. The two set of locations constitute - * the sparse set of matches. These are then used as seeds in the intensification stage of the algorithm. - * @param[in] imgLeft The left Channel of a stereo image. - * @param[in] imgRight The right Channel of a stereo image. - * @param[out] featuresLeft (vector of points) The location of the features in the left image. - * @param[out] featuresRight (vector of points) The location of the features in the right image. - * @note featuresLeft and featuresRight must have the same length and corresponding features - * must be indexed the same way in both vectors. - */ - virtual void sparseMatching(const cv::Mat &imgLeft ,const cv::Mat &imgRight, - std::vector< cv::Point2f > &featuresLeft, - std::vector< cv::Point2f > &featuresRight); - - - /** - * @brief Based on the seeds computed in sparse stereo, this method calculates the semi dense set of correspondences. - * - * The method initially discards low quality matches based on their zero-normalized cross correlation (zncc) value. - * This is done by calling the "extractSparseSeeds" method. Remaining high quality Matches are stored in a t_matchPriorityQueue - * sorted according to their zncc value. The priority queue allows for new matches to be added while keeping track - * of the best Match. The algorithm then process the queue iteratively. - * In every iteration a Match is popped from the queue. The algorithm then tries to find candidate - * matches by matching every point in a small patch around the left Match feature, with a point - * within a same sized patch around the corresponding right feature. For each candidate point match, - * the zncc is computed and if it surpasses a threshold, the candidate pair is stored in a temporary - * priority queue. After this process completed the candidate matches are popped from the Local - * priority queue and if a match is not registered in refMap, it means that is the best match for - * this point. The algorithm registers this point in refMap and also push it to the Seed queue. - * if a candidate match is already registered, it means that is not the best and the algorithm - * discards it. - * - * @note This method does not have input arguments, but uses the "leftFeatures" and "rightFeatures" vectors. - * Also there is no output since the method used refMap and mtcMap to store the results. - * @param[in] featuresLeft The location of the features in the left image. - * @param[in] featuresRight The location of the features in the right image. - */ - void quasiDenseMatching(const std::vector< cv::Point2f > &featuresLeft, - const std::vector< cv::Point2f > &featuresRight); - - - /** - * @brief Compute the disparity map based on the Euclidean distance of corresponding points. - * @param[in] matchMap A matrix of points, the same size as the left channel. Each cell of this - * matrix stores the location of the corresponding point in the right image. - * @param[out] dispMat The disparity map. - * @sa quantizeDisparity - * @sa getDisparity - */ - void computeDisparity(const cv::Mat_ &matchMap, - cv::Mat_ &dispMat); - - - /** - * @brief Disparity map normalization for display purposes. If needed specify the quantization level as input argument. - * @param[in] dispMat The disparity Map. - * @param[in] lvls The quantization level of the output disparity map. - * @return Disparity image. - * @note Stores the output in the disparityImage class variable. - * @sa computeDisparity - * @sa quantiseDisparity - */ - cv::Mat quantiseDisparity(const cv::Mat_ &dispMat, const int lvls); - - - /** - * @brief Compute the Zero-mean Normalized Cross-correlation. - * - * Compare a patch in the left image, centered in point p0 with a patch in the right image, centered in point p1. - * Patches are defined by wy, wx and the patch size is (2*wx+1) by (2*wy+1). - * @param [in] p0 The central point of the patch in the left image. - * @param [in] p1 The central point of the patch in the right image. - * @param [in] wx The distance from the center of the patch to the border in the x direction. - * @param [in] wy The distance from the center of the patch to the border in the y direction. - * @return The value of the the zero-mean normalized cross correlation. - * @note Default value for wx, wy is 1. in this case the patch is 3x3. - */ - float iZNCC_c1(const cv::Point2i p0, const cv::Point2i p1, const int wx=1, const int wy=1); - - - /** - * @brief Compute the sum of values and the sum of squared values of a patch with dimensions - * 2*xWindow+1 by 2*yWindow+1 and centered in point p, using the integral image and integral image of squared pixel values. - * @param[in] p The center of the patch we want to calculate the sum and sum of squared values. - * @param[in] s The integral image - * @param[in] ss The integral image of squared values. - * @param[out] sum The sum of pixels inside the patch. - * @param[out] ssum The sum of squared values inside the patch. - * @param [in] xWindow The distance from the central pixel of the patch to the border in x direction. - * @param [in] yWindow The distance from the central pixel of the patch to the border in y direction. - * @note Default value for xWindow, yWindow is 1. in this case the patch is 3x3. - * @note integral images are very useful to sum values of patches in constant time independent of their - * size. For more information refer to the cv::Integral function OpenCV page. - */ - void patchSumSum2(const cv::Point2i p, const cv::Mat &sum, const cv::Mat &ssum, - float &s, float &ss, const int xWindow=1, const int yWindow=1); - - - /** - * @brief Create a priority queue containing sparse Matches - * - * This method computes the zncc for each Match extracted in "sparseMatching". If the zncc is over - * the correlation threshold then the Match is inserted in the output priority queue. - * @param[in] featuresLeft The feature locations in the left image. - * @param[in] featuresRight The features locations in the right image. - * @param[out] leftMap A matrix of points, of the same size as the left image. Each cell of this - * matrix stores the location of the corresponding point in the right image. - * @param[out] rightMap A matrix of points, the same size as the right image. Each cell of this - * matrix stores the location of the corresponding point in the left image. - * @return Priority queue containing sparse matches. - */ - t_matchPriorityQueue extractSparseSeeds(const std::vector< cv::Point2f > &featuresLeft, - const std::vector< cv::Point2f > &featuresRight, - cv::Mat_ &leftMap, - cv::Mat_ &rightMap); + CV_WRAP virtual cv::Mat getDisparity(uint8_t disparityLvls=50) = 0; - /** - * @brief Check if a match is close to the boarder of an image. - * @param[in] m The match containing points in both image. - * @param[in] bx The offset of the image edge that defines the border in x direction. - * @param[in] by The offset of the image edge that defines the border in y direction. - * @param[in] w The width of the image. - * @param[in] h The height of the image. - * @retval true If the feature is in the border of the image. - * @retval false If the feature is not in the border of image. - */ - bool CheckBorder(Match m, int bx, int by, int w, int h); - + CV_WRAP static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath =""); - /** - * @brief Compare two matches based on their zncc correlation values. - * @param[in] a First Match. - * @param[in] b Second Match. - * @retval true If b is equal or grater Match than a. - * @retval false If b is less Match than a. - */ - bool MatchCompare(const Match a, const Match b); - - - /** - * @brief Build a texture descriptor - * @param[in] img The image we need to compute the descriptor for. - * @param[out] descriptor The texture descriptor of the image. - */ - void buildTextureDescriptor(cv::Mat &img,cv::Mat &descriptor); - - - // Variables used at sparse feature extraction. - // Container for left images' features, extracted with GFT algorithm. - std::vector< cv::Point2f > leftFeatures; - // Container for right images' features, matching is done with LK flow algorithm. - std::vector< cv::Point2f > rightFeatures; - - // Width and height of a single image. - int width; - int height; - int dMatchesLen; - // Containers to store input images. - cv::Mat grayLeft; - cv::Mat grayRight; - // Containers to store the locations of each points pair. - cv::Mat_ refMap; - cv::Mat_ mtcMap; - cv::Mat_ sum0; - cv::Mat_ sum1; - cv::Mat_ ssum0; - cv::Mat_ ssum1; - // Container to store the disparity un-normalized - cv::Mat_ disparity; - // Container to store the disparity image. - cv::Mat_ disparityImg; - // Containers to store textures descriptors. - cv::Mat_ textureDescLeft; - cv::Mat_ textureDescRight; + CV_PROP_RW PropagationParameters Param; }; } //namespace cv diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/qds/src/quasiDenseStereo.cpp index 157a79c6637..d02fbbf74d5 100644 --- a/modules/qds/src/quasiDenseStereo.cpp +++ b/modules/qds/src/quasiDenseStereo.cpp @@ -44,506 +44,670 @@ namespace qds { #define NO_MATCH cv::Point(0,0) -QuasiDenseStereo::QuasiDenseStereo(cv::Size monoImgSize, cv::String paramFilepath) -{ - loadParameters(paramFilepath); - width = monoImgSize.width; - height = monoImgSize.height; - refMap = cv::Mat_(monoImgSize); - mtcMap = cv::Mat_(monoImgSize); - - cv::Size integralSize = cv::Size(monoImgSize.width+1, monoImgSize.height+1); - sum0 = cv::Mat_(integralSize); - sum1 = cv::Mat_(integralSize); - ssum0 = cv::Mat_(integralSize); - ssum1 = cv::Mat_(integralSize); - // the disparity image. - disparity = cv::Mat_(monoImgSize); - disparityImg = cv::Mat_(monoImgSize); - // texture images. - textureDescLeft = cv::Mat_ (monoImgSize); - textureDescRight = cv::Mat_ (monoImgSize); -} - -QuasiDenseStereo::~QuasiDenseStereo() -{ - - rightFeatures.clear(); - leftFeatures.clear(); - - refMap.release(); - mtcMap.release(); - - sum0.release(); - sum1.release(); - ssum0.release(); - ssum1.release(); - // the disparity image. - disparity.release(); - disparityImg.release(); - // texture images. - textureDescLeft.release(); - textureDescRight.release(); -} - -int QuasiDenseStereo::loadParameters(cv::String filepath) +class QuasiDenseStereoImpl : public QuasiDenseStereo { - cv::FileStorage fs; - //if user specified a pathfile, try to use it. - if (!filepath.empty()) +public: + QuasiDenseStereoImpl(cv::Size monoImgSize, cv::String paramFilepath) { - fs.open(filepath, cv::FileStorage::READ); + loadParameters(paramFilepath); + width = monoImgSize.width; + height = monoImgSize.height; + refMap = cv::Mat_(monoImgSize); + mtcMap = cv::Mat_(monoImgSize); + + cv::Size integralSize = cv::Size(monoImgSize.width+1, monoImgSize.height+1); + sum0 = cv::Mat_(integralSize); + sum1 = cv::Mat_(integralSize); + ssum0 = cv::Mat_(integralSize); + ssum1 = cv::Mat_(integralSize); + // the disparity image. + disparity = cv::Mat_(monoImgSize); + disparityImg = cv::Mat_(monoImgSize); + // texture images. + textureDescLeft = cv::Mat_ (monoImgSize); + textureDescRight = cv::Mat_ (monoImgSize); } - // If the file opened, read the parameters. - if (fs.isOpened()) - { - fs["borderX"] >> Param.borderX; - fs["borderY"] >> Param.borderY; - fs["corrWinSizeX"] >> Param.corrWinSizeX; - fs["corrWinSizeY"] >> Param.corrWinSizeY; - fs["correlationThreshold"] >> Param.correlationThreshold; - fs["textrureThreshold"] >> Param.textrureThreshold; - - fs["neighborhoodSize"] >> Param.neighborhoodSize; - fs["disparityGradient"] >> Param.disparityGradient; - - fs["lkTemplateSize"] >> Param.lkTemplateSize; - fs["lkPyrLvl"] >> Param.lkPyrLvl; - fs["lkTermParam1"] >> Param.lkTermParam1; - fs["lkTermParam2"] >> Param.lkTermParam2; - - fs["gftQualityThres"] >> Param.gftQualityThres; - fs["gftMinSeperationDist"] >> Param.gftMinSeperationDist; - fs["gftMaxNumFeatures"] >> Param.gftMaxNumFeatures; - fs.release(); - return 1; - } - // If the filepath was incorrect or non existent, load default parameters. - Param.borderX = 15; - Param.borderY = 15; - // corr window size - Param.corrWinSizeX = 5; - Param.corrWinSizeY = 5; - Param.correlationThreshold = (float)0.5; - Param.textrureThreshold = 200; - - Param.neighborhoodSize = 5; - Param.disparityGradient = 1; - - Param.lkTemplateSize = 3; - Param.lkPyrLvl = 3; - Param.lkTermParam1 = 3; - Param.lkTermParam2 = (float)0.003; - - Param.gftQualityThres = (float)0.01; - Param.gftMinSeperationDist = 10; - Param.gftMaxNumFeatures = 500; - // Return 0 if there was no filepath provides. - // Return -1 if there was a problem opening the filepath provided. - if(filepath.empty()) - { - return 0; - } - return -1; -} -int QuasiDenseStereo::saveParameters(cv::String filepath) -{ - cv::FileStorage fs(filepath, cv::FileStorage::WRITE); - if (fs.isOpened()) + ~QuasiDenseStereoImpl() { - fs << "borderX" << Param.borderX; - fs << "borderY" << Param.borderY; - fs << "corrWinSizeX" << Param.corrWinSizeX; - fs << "corrWinSizeY" << Param.corrWinSizeY; - fs << "correlationThreshold" << Param.correlationThreshold; - fs << "textrureThreshold" << Param.textrureThreshold; - - fs << "neighborhoodSize" << Param.neighborhoodSize; - fs << "disparityGradient" << Param.disparityGradient; - - fs << "lkTemplateSize" << Param.lkTemplateSize; - fs << "lkPyrLvl" << Param.lkPyrLvl; - fs << "lkTermParam1" << Param.lkTermParam1; - fs << "lkTermParam2" << Param.lkTermParam2; - - fs << "gftQualityThres" << Param.gftQualityThres; - fs << "gftMinSeperationDist" << Param.gftMinSeperationDist; - fs << "gftMaxNumFeatures" << Param.gftMaxNumFeatures; - fs.release(); - } - return -1; -} -void QuasiDenseStereo::getSparseMatches(std::vector &sMatches) -{ - Match tmpMatch; - sMatches.clear(); - sMatches.resize(leftFeatures.size()); - for (uint i=0; i &dMatches) -{ - Match tmpMatch; - dMatches.clear(); -// dMatches.resize(dMatchesLen); - for (int row=0; row &featuresLeft, + std::vector< cv::Point2f > &featuresRight) { - for(int col=0; col featureStatus; + std::vector< float > error; + featuresLeft.clear(); + featuresRight.clear(); + + cv::goodFeaturesToTrack(imgLeft, featuresLeft, Param.gftMaxNumFeatures, + Param.gftQualityThres, Param.gftMinSeperationDist); + + cv::Size templateSize(Param.lkTemplateSize,Param.lkTemplateSize); + cv::TermCriteria termination(cv::TermCriteria::MAX_ITER | cv::TermCriteria::EPS, + Param.lkTermParam1, Param.lkTermParam2); + cv::calcOpticalFlowPyrLK(imgLeft, imgRight, featuresLeft, featuresRight, + featureStatus, error, + templateSize, Param.lkPyrLvl, termination); + //discard bad features. + for(size_t i=0; i(row, col); - if (tmpMatch.p1 == NO_MATCH) + if( featureStatus[i]==0 ) { - continue; + std::swap(featuresLeft[i], featuresLeft.back()); + featuresLeft.pop_back(); + std::swap(featureStatus[i], featureStatus.back()); + featureStatus.pop_back(); + std::swap(featuresRight[i], featuresRight.back()); + featuresRight.pop_back(); } - dMatches.push_back(tmpMatch); + else + ++i; } } -} -void QuasiDenseStereo::process(const cv::Mat &imgLeft , const cv::Mat &imgRight) -{ - if (imgLeft.channels()>1) - { - cv::cvtColor(imgLeft, grayLeft, cv::COLOR_BGR2GRAY); - cv::cvtColor(imgRight, grayRight, cv::COLOR_BGR2GRAY); - } - else + + /** + * @brief Based on the seeds computed in sparse stereo, this method calculates the semi dense + * set of correspondences. + * + * The method initially discards low quality matches based on their zero-normalized cross + * correlation (zncc) value. This is done by calling the "extractSparseSeeds" method. Remaining + * high quality Matches stored in a t_matchPriorityQueue sorted according to their zncc value. + * The priority queue allows for new matches to be added while keeping track of the best Match. + * The algorithm then process the queue iteratively. In every iteration a Match is popped from + * the queue. The algorithm then tries to find candidate matches by matching every point in a + * small patch around the left Match feature, with a point within a same sized patch around the + * corresponding right feature. For each candidate point match, the zncc is computed and if it + * surpasses a threshold, the candidate pair is stored in a temporary priority queue. After this + * process completed the candidate matches are popped from the Local priority queue and if a + * match is not registered in refMap, it means that is the best match for this point. The + * algorithm registers this point in refMap and also push it to the Seed queue. If a candidate + * match is already registered, it means that is not the best and the algorithm discards it. + * + * @note This method does not have input arguments, but uses the "leftFeatures" and + * "rightFeatures" vectors. + * Also there is no output since the method used refMap and mtcMap to store the results. + * @param[in] featuresLeft The location of the features in the left image. + * @param[in] featuresRight The location of the features in the right image. + */ + void quasiDenseMatching(const std::vector< cv::Point2f > &featuresLeft, + const std::vector< cv::Point2f > &featuresRight) { - grayLeft = imgLeft.clone(); - grayRight = imgRight.clone(); - } - sparseMatching(grayLeft, grayRight, leftFeatures, rightFeatures); - quasiDenseMatching(leftFeatures, rightFeatures); -} + refMap = cv::Mat_(cv::Size(width, height), cv::Point2i(0, 0)); + mtcMap = cv::Point2i(0, 0); + // build texture homogeneity reference maps. + buildTextureDescriptor(grayLeft, textureDescLeft); + buildTextureDescriptor(grayRight, textureDescRight); -cv::Point2f QuasiDenseStereo::getMatch(const int x, const int y) -{ - return refMap.at(y, x); -} + // generate the intergal images for fast variable window correlation calculations + cv::integral(grayLeft, sum0, ssum0); + cv::integral(grayRight, sum1, ssum1); + // Seed priority queue. The algorithm wants to pop the best seed available in order to densify + //the sparse set. + t_matchPriorityQueue seeds = extractSparseSeeds(featuresLeft, featuresRight, + refMap, mtcMap); -void QuasiDenseStereo::sparseMatching(const cv::Mat &imgLeft ,const cv::Mat &imgRight, - std::vector< cv::Point2f > &featuresLeft, - std::vector< cv::Point2f > &featuresRight) -{ - std::vector< uchar > featureStatus; - std::vector< float > error; - featuresLeft.clear(); - featuresRight.clear(); - - cv::goodFeaturesToTrack(imgLeft, featuresLeft, Param.gftMaxNumFeatures, - Param.gftQualityThres, Param.gftMinSeperationDist); - - cv::Size templateSize(Param.lkTemplateSize,Param.lkTemplateSize); - cv::TermCriteria termination(cv::TermCriteria::MAX_ITER | cv::TermCriteria::EPS, - Param.lkTermParam1, Param.lkTermParam2); - cv::calcOpticalFlowPyrLK(imgLeft, imgRight, featuresLeft, featuresRight, - featureStatus, error, - templateSize, Param.lkPyrLvl, termination); - //discard bad features. - for(size_t i=0; i(p0.y,p0.x) != NO_MATCH) + continue; - m0 /= wa; - m1 /= wa; + // Check the texture descriptor for a boundary + if(textureDescLeft.at(p0.y, p0.x) > Param.textrureThreshold) + continue; - // standard deviations - s0 = sqrt(s0-wa*m0*m0); - s1 = sqrt(s1-wa*m1*m1); + // For all candidate matches. + for(int wy=-Param.disparityGradient; wy<=Param.disparityGradient; wy++) + { + for(int wx=-Param.disparityGradient; wx<=Param.disparityGradient; wx++) + { + cv::Point p1 = cv::Point(m.p1.x+x+wx,m.p1.y+y+wy); + + // Check if its unique in ref + if(mtcMap.at(p1.y, p1.x) != NO_MATCH) + continue; + + // Check the texture descriptor for a boundary + if(textureDescRight.at(p1.y, p1.x) > Param.textrureThreshold) + continue; + + // Calculate ZNCC and store local match. + float corr = iZNCC_c1(p0,p1,Param.corrWinSizeX,Param.corrWinSizeY); + + // push back if this is valid match + if( corr > Param.correlationThreshold ) + { + Match nm; + nm.p0 = p0; + nm.p1 = p1; + nm.corr = corr; + Local.push(nm); + } + } + } + } + } + + // Get seeds from the local + while( !Local.empty() ) + { + Match lm = Local.top(); + Local.pop(); + // Check if its unique in both ref and dst. + if(refMap.at(lm.p0.y, lm.p0.x) != NO_MATCH) + continue; + if(mtcMap.at(lm.p1.y, lm.p1.x) != NO_MATCH) + continue; + + + // Unique match + refMap.at(lm.p0.y, lm.p0.x) = lm.p1; + mtcMap.at(lm.p1.y, lm.p1.x) = lm.p0; + // Add to the seed list + seeds.push(lm); + } + } + } - for (int col=-wy; col<=wy; col++) + /** + * @brief Compute the disparity map based on the Euclidean distance of corresponding points. + * @param[in] matchMap A matrix of points, the same size as the left channel. Each cell of this + * matrix stores the location of the corresponding point in the right image. + * @param[out] dispMat The disparity map. + * @sa quantizeDisparity + * @sa getDisparity + */ + void computeDisparity(const cv::Mat_ &matchMap, + cv::Mat_ &dispMat) { - for (int row=-wx; row<=wx; row++) + for(int row=0; row< height; row++) { - zncc += (float)grayLeft.at(p0.y+row, p0.x+col) * - (float)grayRight.at(p1.y+row, p1.x+col); + for(int col=0; col(tmpPoint) == NO_MATCH) + { + dispMat.at(tmpPoint) = 200; + continue; + } + //if a match is found, compute the difference in location of the match and current + //pixel. + int dx = col-matchMap.at(tmpPoint).x; + int dy = row-matchMap.at(tmpPoint).y; + //calculate disparity of current pixel. + dispMat.at(tmpPoint) = sqrt(float(dx*dx+dy*dy)); + } } } - zncc = (zncc-wa*m0*m1)/(s0*s1); - return zncc; -} -void QuasiDenseStereo::patchSumSum2(const cv::Point2i p, const cv::Mat &sum, const cv::Mat &ssum, - float &s, float &ss, const int xWindow, const int yWindow) -{ - cv::Point2i otl(p.x-xWindow, p.y-yWindow); - //outer top right - cv::Point2i otr(p.x+xWindow+1, p.y-yWindow); - //outer bottom left - cv::Point2i obl(p.x-xWindow, p.y+yWindow+1); - //outer bottom right - cv::Point2i obr(p.x+xWindow+1, p.y+yWindow+1); - - // sum and squared sum for right window - s = (float)(sum.at(otl) - sum.at(otr) - sum.at(obl) + sum.at(obr)); - - ss = (float)(ssum.at(otl) - ssum.at(otr) - ssum.at(obl) + ssum.at(obr)); -} + /** + * @brief Disparity map normalization for display purposes. If needed specify the quantization + * level as input argument. + * @param[in] dispMat The disparity Map. + * @param[in] lvls The quantization level of the output disparity map. + * @return Disparity image. + * @note Stores the output in the disparityImage class variable. + * @sa computeDisparity + * @sa getDisparity + */ + cv::Mat quantiseDisparity(const cv::Mat_ &dispMat, const int lvls) + { + float tmpPixelVal ; + double min, max; +// minMaxLoc(disparity, &min, &max); + min = 0; + max = lvls; + for(int row=0; row(row, col); + tmpPixelVal = (float) (255. - 255.0*(tmpPixelVal-min)/(max-min)); + + disparityImg.at(row, col) = (uint8_t) tmpPixelVal; + } + } + return disparityImg; + } + + + /** + * @brief Compute the Zero-mean Normalized Cross-correlation. + * + * Compare a patch in the left image, centered in point p0 with a patch in the right image, + * centered in point p1. Patches are defined by wy, wx and the patch size is (2*wx+1) by + * (2*wy+1). + * @param [in] p0 The central point of the patch in the left image. + * @param [in] p1 The central point of the patch in the right image. + * @param [in] wx The distance from the center of the patch to the border in the x direction. + * @param [in] wy The distance from the center of the patch to the border in the y direction. + * @return The value of the the zero-mean normalized cross correlation. + * @note Default value for wx, wy is 1. in this case the patch is 3x3. + */ + float iZNCC_c1(const cv::Point2i p0, const cv::Point2i p1, const int wx=1, const int wy=1) + { + float m0=0.0 ,m1=0.0 ,s0=0.0 ,s1=0.0; + float wa = (float)(2*wy+1)*(2*wx+1); + float zncc=0.0; -void QuasiDenseStereo::buildTextureDescriptor(cv::Mat &src,cv::Mat &descriptor) -{ + patchSumSum2(p0, sum0, ssum0, m0, s0, wx, wy); + patchSumSum2(p1, sum1, ssum1, m1, s1, wx, wy); - float a, b, c, d; + m0 /= wa; + m1 /= wa; - //for each pixel in the input image. The boundaries are this way not to raise errors when computing a,b,c,d? + // standard deviations + s0 = sqrt(s0-wa*m0*m0); + s1 = sqrt(s1-wa*m1*m1); - uint8_t center, top, bottom, right, left; - //reset descriptors - // traverse every pixel. - for(int row=1; row(row,col); - top = src.at(row-1,col); - bottom = src.at(row+1,col); - left = src.at(row,col-1); - right = src.at(row,col+1); - - a = (float)abs(center - top); - b = (float)abs(center - bottom); - c = (float)abs(center - left); - d = (float)abs(center - right); - //choose the biggest of them. - int val = (int) std::max(a, std::max(b, std::max(c, d))); - descriptor.at(row, col) = val; + for (int row=-wx; row<=wx; row++) + { + zncc += (float)grayLeft.at(p0.y+row, p0.x+col) * + (float)grayRight.at(p1.y+row, p1.x+col); + } } + zncc = (zncc-wa*m0*m1)/(s0*s1); + return zncc; } -} -bool QuasiDenseStereo::CheckBorder(Match m, int bx, int by, int w, int h) -{ - if(m.p0.xw-bx || m.p0.yh-by || - m.p1.xw-bx || m.p1.yh-by) + + /** + * @brief Compute the sum of values and the sum of squared values of a patch with dimensions + * 2*xWindow+1 by 2*yWindow+1 and centered in point p, using the integral image and integral + * image of squared pixel values. + * @param[in] p The center of the patch we want to calculate the sum and sum of squared values. + * @param[in] s The integral image + * @param[in] ss The integral image of squared values. + * @param[out] sum The sum of pixels inside the patch. + * @param[out] ssum The sum of squared values inside the patch. + * @param [in] xWindow The distance from the central pixel of the patch to the border in x + * direction. + * @param [in] yWindow The distance from the central pixel of the patch to the border in y + * direction. + * @note Default value for xWindow, yWindow is 1. in this case the patch is 3x3. + * @note integral images are very useful to sum values of patches in constant time independent + * of their size. For more information refer to the cv::Integral function OpenCV page. + */ + void patchSumSum2(const cv::Point2i p, const cv::Mat &sum, const cv::Mat &ssum, + float &s, float &ss, const int xWindow=1, const int yWindow=1) { - return false; + cv::Point2i otl(p.x-xWindow, p.y-yWindow); + //outer top right + cv::Point2i otr(p.x+xWindow+1, p.y-yWindow); + //outer bottom left + cv::Point2i obl(p.x-xWindow, p.y+yWindow+1); + //outer bottom right + cv::Point2i obr(p.x+xWindow+1, p.y+yWindow+1); + + // sum and squared sum for right window + s = (float)(sum.at(otl) - sum.at(otr) + - sum.at(obl) + sum.at(obr)); + + ss = (float)(ssum.at(otl) - ssum.at(otr) + - ssum.at(obl) + ssum.at(obr)); } - return true; -} -bool QuasiDenseStereo::MatchCompare(const Match a, const Match b) -{ - if(a.corr<=b.corr)return true; - return false; -} - -t_matchPriorityQueue QuasiDenseStereo::extractSparseSeeds(const std::vector< cv::Point2f > &featuesLeft, - const std::vector< cv::Point2f > &featuresRight, - cv::Mat_ &leftMap, - cv::Mat_ &rightMap) -{ - t_matchPriorityQueue seeds; - for(uint i=0; i < featuesLeft.size(); i++) + /** + * @brief Create a priority queue containing sparse Matches + * + * This method computes the zncc for each Match extracted in "sparseMatching". If the zncc is + * over the correlation threshold then the Match is inserted in the output priority queue. + * @param[in] featuresLeft The feature locations in the left image. + * @param[in] featuresRight The features locations in the right image. + * @param[out] leftMap A matrix of points, of the same size as the left image. Each cell of this + * matrix stores the location of the corresponding point in the right image. + * @param[out] rightMap A matrix of points, the same size as the right image. Each cell of this + * matrix stores the location of the corresponding point in the left image. + * @return Priority queue containing sparse matches. + */ + t_matchPriorityQueue extractSparseSeeds(const std::vector< cv::Point2f > &featuresLeft, + const std::vector< cv::Point2f > &featuresRight, + cv::Mat_ &leftMap, + cv::Mat_ &rightMap) { - // Calculate correlation and store match in Seeds. - Match m; - m.p0 = cv::Point2i(featuesLeft[i]); - m.p1 = cv::Point2i(featuresRight[i]); - m.corr = 0; - - // Check if too close to boundary. - if(!CheckBorder(m,Param.borderX,Param.borderY, width, height)) - continue; - - m.corr = iZNCC_c1(m.p0, m.p1, Param.corrWinSizeX, Param.corrWinSizeY); - // Can we add it to the list - if( m.corr > Param.correlationThreshold ) + t_matchPriorityQueue seeds; + for(uint i=0; i < featuresLeft.size(); i++) { - seeds.push(m); - leftMap.at(m.p0.y, m.p0.x) = m.p1; - rightMap.at(m.p1.y, m.p1.x) = m.p0; + // Calculate correlation and store match in Seeds. + Match m; + m.p0 = cv::Point2i(featuresLeft[i]); + m.p1 = cv::Point2i(featuresRight[i]); + m.corr = 0; + + // Check if too close to boundary. + if(!CheckBorder(m,Param.borderX,Param.borderY, width, height)) + continue; + + m.corr = iZNCC_c1(m.p0, m.p1, Param.corrWinSizeX, Param.corrWinSizeY); + // Can we add it to the list + if( m.corr > Param.correlationThreshold ) + { + seeds.push(m); + leftMap.at(m.p0.y, m.p0.x) = m.p1; + rightMap.at(m.p1.y, m.p1.x) = m.p0; + } } + return seeds; } - return seeds; -} -void QuasiDenseStereo::quasiDenseMatching(const std::vector< cv::Point2f > &featuresLeft, - const std::vector< cv::Point2f > &featuresRight) -{ - refMap = cv::Mat_(cv::Size(width, height), cv::Point2i(0, 0)); - mtcMap = cv::Point2i(0, 0); - // build texture homogeneity reference maps. - buildTextureDescriptor(grayLeft, textureDescLeft); - buildTextureDescriptor(grayRight, textureDescRight); - - // generate the intergal images for fast variable window correlation calculations - cv::integral(grayLeft, sum0, ssum0); - cv::integral(grayRight, sum1, ssum1); + /** + * @brief Check if a match is close to the boarder of an image. + * @param[in] m The match containing points in both image. + * @param[in] bx The offset of the image edge that defines the border in x direction. + * @param[in] by The offset of the image edge that defines the border in y direction. + * @param[in] w The width of the image. + * @param[in] h The height of the image. + * @retval true If the feature is in the border of the image. + * @retval false If the feature is not in the border of image. + */ + bool CheckBorder(Match m, int bx, int by, int w, int h) + { + if(m.p0.xw-bx || m.p0.yh-by || + m.p1.xw-bx || m.p1.yh-by) + { + return false; + } - // Seed priority queue. The algorithm wants to pop the best seed available in order to densify the mess. // Evs mess ??? the sparse set maybe better ?? !!!!!!! - t_matchPriorityQueue seeds = extractSparseSeeds(featuresLeft, featuresRight, - refMap, mtcMap); + return true; + } - // Do the propagation part - while(!seeds.empty()) + /** + * @brief Build a texture descriptor + * @param[in] img The image we need to compute the descriptor for. + * @param[out] descriptor The texture descriptor of the image. + */ + void buildTextureDescriptor(cv::Mat &img,cv::Mat &descriptor) { - t_matchPriorityQueue Local; + float a, b, c, d; - // Get the best seed at the moment - Match m = seeds.top(); - seeds.pop(); + uint8_t center, top, bottom, right, left; + //reset descriptors - // Ignore the border - if(!CheckBorder(m, Param.borderX, Param.borderY, width, height)) - continue; - - // For all neighbours of the seed in image 1 - //the neighborghoud is defined with Param.N*2 dimentrion - for(int y=-Param.neighborhoodSize;y<=Param.neighborhoodSize;y++) + // traverse every pixel. + for(int row=1; row(p0.y,p0.x) != NO_MATCH) - continue; - - // Check the texture descriptor for a boundary - if(textureDescLeft.at(p0.y, p0.x) > Param.textrureThreshold) - continue; - - // For all candidate matches. - for(int wy=-Param.disparityGradient; wy<=Param.disparityGradient; wy++) - { - for(int wx=-Param.disparityGradient; wx<=Param.disparityGradient; wx++) - { - cv::Point p1 = cv::Point(m.p1.x+x+wx,m.p1.y+y+wy); - - // Check if its unique in ref - if(mtcMap.at(p1.y, p1.x) != NO_MATCH) - continue; - - // Check the texture descriptor for a boundary - if(textureDescRight.at(p1.y, p1.x) > Param.textrureThreshold) - continue; - - // Calculate ZNCC and store local match. - float corr = iZNCC_c1(p0,p1,Param.corrWinSizeX,Param.corrWinSizeY); - - // push back if this is valid match - if( corr > Param.correlationThreshold ) - { - Match nm; - nm.p0 = p0; - nm.p1 = p1; - nm.corr = corr; - Local.push(nm); - } - } - } + // the values of the current pixel. + center = img.at(row,col); + top = img.at(row-1,col); + bottom = img.at(row+1,col); + left = img.at(row,col-1); + right = img.at(row,col+1); + + a = (float)abs(center - top); + b = (float)abs(center - bottom); + c = (float)abs(center - left); + d = (float)abs(center - right); + //choose the biggest of them. + int val = (int) std::max(a, std::max(b, std::max(c, d))); + descriptor.at(row, col) = val; } } + } - // Get seeds from the local - while( !Local.empty() ) - { - Match lm = Local.top(); - Local.pop(); - // Check if its unique in both ref and dst. - if(refMap.at(lm.p0.y, lm.p0.x) != NO_MATCH) - continue; - if(mtcMap.at(lm.p1.y, lm.p1.x) != NO_MATCH) - continue; + //------------------------------------------------------------------------- - // Unique match - refMap.at(lm.p0.y, lm.p0.x) = lm.p1; - mtcMap.at(lm.p1.y, lm.p1.x) = lm.p0; - // Add to the seed list - seeds.push(lm); + void getSparseMatches(std::vector &sMatches) + { + Match tmpMatch; + sMatches.clear(); + sMatches.resize(leftFeatures.size()); + for (uint i=0; i> Param.borderX; + fs["borderY"] >> Param.borderY; + fs["corrWinSizeX"] >> Param.corrWinSizeX; + fs["corrWinSizeY"] >> Param.corrWinSizeY; + fs["correlationThreshold"] >> Param.correlationThreshold; + fs["textrureThreshold"] >> Param.textrureThreshold; + + fs["neighborhoodSize"] >> Param.neighborhoodSize; + fs["disparityGradient"] >> Param.disparityGradient; + + fs["lkTemplateSize"] >> Param.lkTemplateSize; + fs["lkPyrLvl"] >> Param.lkPyrLvl; + fs["lkTermParam1"] >> Param.lkTermParam1; + fs["lkTermParam2"] >> Param.lkTermParam2; + + fs["gftQualityThres"] >> Param.gftQualityThres; + fs["gftMinSeperationDist"] >> Param.gftMinSeperationDist; + fs["gftMaxNumFeatures"] >> Param.gftMaxNumFeatures; + fs.release(); + return 1; + } + // If the filepath was incorrect or non existent, load default parameters. + Param.borderX = 15; + Param.borderY = 15; + // corr window size + Param.corrWinSizeX = 5; + Param.corrWinSizeY = 5; + Param.correlationThreshold = (float)0.5; + Param.textrureThreshold = 200; + + Param.neighborhoodSize = 5; + Param.disparityGradient = 1; + + Param.lkTemplateSize = 3; + Param.lkPyrLvl = 3; + Param.lkTermParam1 = 3; + Param.lkTermParam2 = (float)0.003; + + Param.gftQualityThres = (float)0.01; + Param.gftMinSeperationDist = 10; + Param.gftMaxNumFeatures = 500; + // Return 0 if there was no filepath provides. + // Return -1 if there was a problem opening the filepath provided. + if(filepath.empty()) + { + return 0; + } + return -1; + } -void QuasiDenseStereo::computeDisparity(const cv::Mat_ &matchMap, - cv::Mat_ &dispMat) -{ - for(int row=0; row< height; row++) + int saveParameters(cv::String filepath) { - for(int col=0; col(tmpPoint) == NO_MATCH) + void getDenseMatches(std::vector &dMatches) + { + Match tmpMatch; + dMatches.clear(); + // dMatches.resize(dMatchesLen); + for (int row=0; row(tmpPoint) = 200; - continue; + tmpMatch.p0 = cv::Point(col, row); + tmpMatch.p1 = refMap.at(row, col); + if (tmpMatch.p1 == NO_MATCH) + { + continue; + } + dMatches.push_back(tmpMatch); } - //if a match is found, compute the difference in location of the match and current pixel. - int dx = col-matchMap.at(tmpPoint).x; - int dy = row-matchMap.at(tmpPoint).y; - //calculate disparity of current pixel. - dispMat.at(tmpPoint) = sqrt(float(dx*dx+dy*dy)); } } -} -cv::Mat QuasiDenseStereo::quantiseDisparity(const cv::Mat_ &dispMat, const int lvls) -{ - float tmpPixelVal ; - double min, max; -// cv::minMaxLoc(disparity, &min, &max); - min = 0; - max = lvls; - for(int row=0; row1) { - tmpPixelVal = dispMat.at(row, col); - tmpPixelVal = (float) (255. - 255.0*(tmpPixelVal-min)/(max-min)); - - disparityImg.at(row, col) = (uint8_t) tmpPixelVal; + cv::cvtColor(imgLeft, grayLeft, cv::COLOR_BGR2GRAY); + cv::cvtColor(imgRight, grayRight, cv::COLOR_BGR2GRAY); + } + else + { + grayLeft = imgLeft.clone(); + grayRight = imgRight.clone(); } + sparseMatching(grayLeft, grayRight, leftFeatures, rightFeatures); + quasiDenseMatching(leftFeatures, rightFeatures); + } + + + cv::Point2f getMatch(const int x, const int y) + { + return refMap.at(y, x); + } + + + cv::Mat getDisparity(uint8_t disparityLvls) + { + computeDisparity(refMap, disparity); + return quantiseDisparity(disparity, disparityLvls); } - return disparityImg; -} -cv::Mat QuasiDenseStereo::getDisparity(uint8_t disparityLvls) + // Variables used at sparse feature extraction. + // Container for left images' features, extracted with GFT algorithm. + std::vector< cv::Point2f > leftFeatures; + // Container for right images' features, matching is done with LK flow algorithm. + std::vector< cv::Point2f > rightFeatures; + + // Width and height of a single image. + int width; + int height; + int dMatchesLen; + // Containers to store input images. + cv::Mat grayLeft; + cv::Mat grayRight; + // Containers to store the locations of each points pair. + cv::Mat_ refMap; + cv::Mat_ mtcMap; + cv::Mat_ sum0; + cv::Mat_ sum1; + cv::Mat_ ssum0; + cv::Mat_ ssum1; + // Container to store the disparity un-normalized + cv::Mat_ disparity; + // Container to store the disparity image. + cv::Mat_ disparityImg; + // Containers to store textures descriptors. + cv::Mat_ textureDescLeft; + cv::Mat_ textureDescRight; + +}; + +cv::Ptr QuasiDenseStereo::create(cv::Size monoImgSize, cv::String paramFilepath) { - computeDisparity(refMap, disparity); - return quantiseDisparity(disparity, disparityLvls); + return cv::makePtr(monoImgSize, paramFilepath); } +QuasiDenseStereo::~QuasiDenseStereo(){ + +} + + } } From 1d929601cbdf6c2ee2b8716fe71c8175692e4425 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Mon, 17 Dec 2018 19:22:22 +0000 Subject: [PATCH 14/29] Fix to comply with new public API. --- modules/qds/samples/dense_disparity.cpp | 8 ++++---- modules/qds/samples/export_param_file.cpp | 8 +++++--- modules/qds/tutorials/export_param_file.markdown | 12 ++++++------ modules/qds/tutorials/quasi_dense_stereo.markdown | 10 +++++----- 4 files changed, 20 insertions(+), 18 deletions(-) diff --git a/modules/qds/samples/dense_disparity.cpp b/modules/qds/samples/dense_disparity.cpp index ff1e0419d0f..b2284513dc0 100644 --- a/modules/qds/samples/dense_disparity.cpp +++ b/modules/qds/samples/dense_disparity.cpp @@ -20,19 +20,19 @@ int main() rightImg = imread("./imgRight.png", IMREAD_COLOR); cv::Size frameSize = leftImg.size(); // Initialize qds and start process. - qds::QuasiDenseStereo stereo(frameSize); + Ptr stereo = qds::QuasiDenseStereo::create(frameSize); uint8_t displvl = 80; // Number of disparity levels cv::Mat disp; // Compute dense stereo. - stereo.process(leftImg, rightImg); + stereo->process(leftImg, rightImg); // Compute disparity between left and right channel of current frame. - disp = stereo.getDisparity(displvl); + disp = stereo->getDisparity(displvl); vector matches; - stereo.getDenseMatches(matches); + stereo->getDenseMatches(matches); // Create three windows and show images. cv::namedWindow("right channel"); diff --git a/modules/qds/samples/export_param_file.cpp b/modules/qds/samples/export_param_file.cpp index 56f550f287b..96f41f9efca 100644 --- a/modules/qds/samples/export_param_file.cpp +++ b/modules/qds/samples/export_param_file.cpp @@ -6,11 +6,13 @@ using namespace std; int main(int argc, char* argv[]) { - std::string parameterFileLocation = ""; + std::string parameterFileLocation = "./parameters.yaml"; if (argc > 1) - parameterFileLocation = argv[1]; + parameterFileLocation = argv[1]; - qds::QuasiDenseStereo(cv::Size(5,5)).saveParameters(parameterFileLocation); + + Ptr stereo = qds::QuasiDenseStereo::create(cv::Size(5,5)); + stereo->saveParameters(parameterFileLocation); return 0; } diff --git a/modules/qds/tutorials/export_param_file.markdown b/modules/qds/tutorials/export_param_file.markdown index 8ceb35ad8b3..cd202b0e714 100644 --- a/modules/qds/tutorials/export_param_file.markdown +++ b/modules/qds/tutorials/export_param_file.markdown @@ -14,13 +14,13 @@ In this tutorial you will learn how to ## Explanation: The class supports loading configuration parameters from a .yaml file using the method `loadParameters()`. -This is very useful for fine-tuning the class' parameters on the fly. To extract a template of this parameter -file you just run 2 lines of code. +This is very useful for fine-tuning the class' parameters on the fly. To extract a template of this +parameter file you run the following code. ``` -parameterFileLocation = "./parameters.yaml" -qds::QuasiDenseStereo(cv::Size(5,5)).saveParameters(parameterFileLocation); - +std::string parameterFileLocation = "./parameters.yaml"; +Ptr stereo = qds::QuasiDenseStereo::create(cv::Size(5,5)); +stereo->saveParameters(parameterFileLocation); ``` We make an instance of a `QuasiDenseStereo` object. Not specifying the second argument of the constructor, -makes the object to load default parameters from [default.h](../include/opencv2/qds/defaults.hpp). +makes the object to load default parameters. By calling the method `saveParameters()`, we store the template file to the location specified by `parameterFileLocation` diff --git a/modules/qds/tutorials/quasi_dense_stereo.markdown b/modules/qds/tutorials/quasi_dense_stereo.markdown index a964b694069..abcf6091350 100644 --- a/modules/qds/tutorials/quasi_dense_stereo.markdown +++ b/modules/qds/tutorials/quasi_dense_stereo.markdown @@ -26,14 +26,14 @@ rightImg = imread("./imgRight.png", IMREAD_COLOR); We need to know the frame size of a single image, in order to create an instance of a `QuasiDesnseStereo` object. ``` cv::Size frameSize = leftImg.size(); -qds::QuasiDenseStereo stereo(frameSize); +Ptr stereo = qds::QuasiDenseStereo::create(frameSize); ``` Because we didn't specify the second argument in the constructor, the `QuasiDesnseStereo` object will -load default parameters from this [headerfile](../include/opencv2/qds/defaults.hpp). +load default parameters. We can then pass the imported stereo images in the process method like this ``` -stereo.process(leftChannel, rightChannel); +stereo->process(leftImg, rightImg); ``` The process method contains most of the functionality of the class and does two main things. - Computes a sparse stereo based in "Good Features to Track" and "pyramidal Lucas-Kanade" flow algorithm @@ -43,7 +43,7 @@ After the execution of `process()` we can display the disparity Image of the ste ``` int displvl = 80; Mat disp; -disp = stereo.getDisparity(displvl); +disp = stereo->getDisparity(displvl); cv::namedWindow("disparity map"); cv::imshow("disparity map", disp); ``` @@ -51,5 +51,5 @@ cv::imshow("disparity map", disp); At this point we can also extract all the corresponding points using `getDenseMatches()` method. ``` vector matches; -stereo.getDenseMatches(matches); +stereo->getDenseMatches(matches); ``` From b81a070d4f08e285d47dcdfeda64e2d8bca32d3d Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Mon, 17 Dec 2018 19:24:20 +0000 Subject: [PATCH 15/29] Remove unnecessary modules --- modules/qds/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/qds/CMakeLists.txt b/modules/qds/CMakeLists.txt index 260ff2c16ee..0f444e049ef 100644 --- a/modules/qds/CMakeLists.txt +++ b/modules/qds/CMakeLists.txt @@ -1,3 +1,3 @@ set(the_description "Quasi Dense Stereo") -ocv_define_module(qds opencv_core opencv_imgproc opencv_video opencv_imgcodecs opencv_tracking OPTIONAL opencv_video opencv_imgproc opencv_imgcodecs opencv_core WRAP python) +ocv_define_module(qds opencv_core opencv_imgproc opencv_video opencv_tracking WRAP python) From e3c9eecb391bf24066564b01706dc1b3a6155323 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Mon, 17 Dec 2018 23:21:32 +0000 Subject: [PATCH 16/29] Fix maybe-uninitialized warnings on linux --- modules/qds/src/quasiDenseStereo.cpp | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/qds/src/quasiDenseStereo.cpp index d02fbbf74d5..d05da5f9390 100644 --- a/modules/qds/src/quasiDenseStereo.cpp +++ b/modules/qds/src/quasiDenseStereo.cpp @@ -519,7 +519,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo //------------------------------------------------------------------------- - void getSparseMatches(std::vector &sMatches) + void getSparseMatches(std::vector &sMatches) override { Match tmpMatch; sMatches.clear(); @@ -531,7 +531,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo sMatches.push_back(tmpMatch); } } - int loadParameters(cv::String filepath) + int loadParameters(cv::String filepath) override { cv::FileStorage fs; //if user specified a pathfile, try to use it. @@ -592,7 +592,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo return -1; } - int saveParameters(cv::String filepath) + int saveParameters(cv::String filepath) override { cv::FileStorage fs(filepath, cv::FileStorage::WRITE); if (fs.isOpened()) @@ -620,7 +620,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo return -1; } - void getDenseMatches(std::vector &dMatches) + void getDenseMatches(std::vector &dMatches) override { Match tmpMatch; dMatches.clear(); @@ -640,7 +640,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo } } - void process(const cv::Mat &imgLeft , const cv::Mat &imgRight) + void process(const cv::Mat &imgLeft , const cv::Mat &imgRight) override { if (imgLeft.channels()>1) { @@ -657,13 +657,13 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo } - cv::Point2f getMatch(const int x, const int y) + cv::Point2f getMatch(const int x, const int y) override { return refMap.at(y, x); } - cv::Mat getDisparity(uint8_t disparityLvls) + cv::Mat getDisparity(uint8_t disparityLvls) override { computeDisparity(refMap, disparity); return quantiseDisparity(disparity, disparityLvls); From 2738d12b97f2e3c0bb21ed3a1fd9e4d0cd11e1f6 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Mon, 17 Dec 2018 23:40:51 +0000 Subject: [PATCH 17/29] Migration to stereo module --- modules/README.md | 4 +- modules/qds/CMakeLists.txt | 3 - modules/qds/README.md | 8 -- modules/qds/include/opencv2/qds.hpp | 73 ------------------- .../misc/python/pyopencv_quasiDenseStereo.hpp | 17 ----- modules/qds/src/precomp.hpp | 47 ------------ modules/stereo/CMakeLists.txt | 2 +- modules/stereo/README.md | 7 ++ .../doc/qds.bib => stereo/doc/stereo.bib} | 0 modules/stereo/include/opencv2/stereo.hpp | 1 + .../opencv2/stereo/quasi_dense_stereo.hpp} | 46 ++++++++---- .../samples/dense_disparity.cpp | 6 +- .../samples/export_param_file.cpp | 4 +- .../src/quasi_dense_stereo.cpp} | 10 ++- .../tutorials/export_param_file.markdown | 2 +- .../tutorials/quasi_dense_stereo.markdown | 4 +- 16 files changed, 57 insertions(+), 177 deletions(-) delete mode 100644 modules/qds/CMakeLists.txt delete mode 100644 modules/qds/README.md delete mode 100644 modules/qds/include/opencv2/qds.hpp delete mode 100644 modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp delete mode 100644 modules/qds/src/precomp.hpp rename modules/{qds/doc/qds.bib => stereo/doc/stereo.bib} (100%) rename modules/{qds/include/opencv2/qds/quasiDenseStereo.hpp => stereo/include/opencv2/stereo/quasi_dense_stereo.hpp} (75%) rename modules/{qds => stereo}/samples/dense_disparity.cpp (89%) rename modules/{qds => stereo}/samples/export_param_file.cpp (70%) rename modules/{qds/src/quasiDenseStereo.cpp => stereo/src/quasi_dense_stereo.cpp} (98%) rename modules/{qds => stereo}/tutorials/export_param_file.markdown (90%) rename modules/{qds => stereo}/tutorials/quasi_dense_stereo.markdown (93%) diff --git a/modules/README.md b/modules/README.md index 7bdce47c62c..26387d2e34c 100644 --- a/modules/README.md +++ b/modules/README.md @@ -46,8 +46,6 @@ $ cmake -D OPENCV_EXTRA_MODULES_PATH=/modules -D BUILD_opencv_/modules -D BUILD_opencv_ - -/** - * @defgroup qds Quasi Dense Stereo - * This module contains the code to perform quasi dense stereo matching. - * The method initially starts with a sparse 3D reconstruction based on feature matching across a - * stereo image pair and subsequently propagates the structure into neighboring image regions. - * To obtain initial seed correspondences, the algorithm locates Shi and Tomashi features in the - * left image of the stereo pair and then tracks them using pyramidal Lucas-Kanade in the right image. - * To densify the sparse correspondences, the algorithm computes the zero-mean normalized - * cross-correlation (ZNCC) in small patches around every seed pair and uses it as a quality metric - * for each match. In this code, we introduce a custom structure to store the location and ZNCC value - * of correspondences called "Match". Seed Matches are stored in a priority queue sorted according to - * their ZNCC value, allowing for the best quality Match to be readily available. The algorithm pops - * Matches and uses them to extract new matches around them. This is done by considering a small - * neighboring area around each Seed and retrieving correspondences above a certain texture threshold - * that are not previously computed. New matches are stored in the seed priority queue and used as seeds. - * The propagation process ends when no additional matches can be retrieved. - * - * - * @sa This code represents the work presented in @cite Stoyanov2010. - * If this code is useful for your work please cite @cite Stoyanov2010. - * - * Also the original growing scheme idea is described in @cite Lhuillier2000 - * -*/ -#endif // __OPENCV_QDS_H__ diff --git a/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp b/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp deleted file mode 100644 index 30adc97082c..00000000000 --- a/modules/qds/misc/python/pyopencv_quasiDenseStereo.hpp +++ /dev/null @@ -1,17 +0,0 @@ -#ifdef HAVE_OPENCV_QDS - -template<> struct pyopencvVecConverter -{ - static bool to(PyObject* obj, std::vector& value, const ArgInfo info) - { - return pyopencv_to_generic_vec(obj, value, info); - } - - static PyObject* from(const std::vector& value) - { - return pyopencv_from_generic_vec(value); - } -}; -typedef std::vector vector_qds_Match; - -#endif diff --git a/modules/qds/src/precomp.hpp b/modules/qds/src/precomp.hpp deleted file mode 100644 index 5d355a2998e..00000000000 --- a/modules/qds/src/precomp.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/* -By downloading, copying, installing or using the software you agree to this license. -If you do not agree to this license, do not download, install, -copy or use the software. - - - License Agreement - For Open Source Computer Vision Library - (3-clause BSD License) - -Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. -Third party copyrights are property of their respective owners. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the names of the copyright holders nor the names of the contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -This software is provided by the copyright holders and contributors "as is" and -any express or implied warranties, including, but not limited to, the implied -warranties of merchantability and fitness for a particular purpose are disclaimed. -In no event shall copyright holders or contributors be liable for any direct, -indirect, incidental, special, exemplary, or consequential damages -(including, but not limited to, procurement of substitute goods or services; -loss of use, data, or profits; or business interruption) however caused -and on any theory of liability, whether in contract, strict liability, -or tort (including negligence or otherwise) arising in any way out of -the use of this software, even if advised of the possibility of such damage. - */ - -#ifndef __OPENCV_PRECOMP_H__ -#define __OPENCV_PRECOMP_H__ - -#include -#include -#include - -#endif diff --git a/modules/stereo/CMakeLists.txt b/modules/stereo/CMakeLists.txt index 97297e879d4..25412b6a12b 100644 --- a/modules/stereo/CMakeLists.txt +++ b/modules/stereo/CMakeLists.txt @@ -1,2 +1,2 @@ set(the_description "Stereo Correspondence") -ocv_define_module(stereo opencv_imgproc opencv_features2d opencv_core opencv_calib3d) +ocv_define_module(stereo opencv_imgproc opencv_features2d opencv_core opencv_calib3d opencv_tracking opencv_video) diff --git a/modules/stereo/README.md b/modules/stereo/README.md index 745064dbd60..f12f37dcfef 100644 --- a/modules/stereo/README.md +++ b/modules/stereo/README.md @@ -2,3 +2,10 @@ Stereo Correspondence with different descriptors ================================================ Stereo matching done with different descriptors: Census / CS-Census / MCT / BRIEF / MV. + +Quasi Dense Stereo +====================== + +Quasi Dense Stereo is method for performing dense stereo matching. +The code uses pyramidal Lucas-Kanade with Shi-Tomasi features to get the initial seed correspondences. +Then these seeds are propagated by using mentioned growing scheme. diff --git a/modules/qds/doc/qds.bib b/modules/stereo/doc/stereo.bib similarity index 100% rename from modules/qds/doc/qds.bib rename to modules/stereo/doc/stereo.bib diff --git a/modules/stereo/include/opencv2/stereo.hpp b/modules/stereo/include/opencv2/stereo.hpp index fc22938b943..5eea8058e65 100644 --- a/modules/stereo/include/opencv2/stereo.hpp +++ b/modules/stereo/include/opencv2/stereo.hpp @@ -49,6 +49,7 @@ #include "opencv2/core/affine.hpp" #include "opencv2/stereo/descriptor.hpp" #include "opencv2/stereo/matching.hpp" +#include /** @defgroup stereo Stereo Correspondance Algorithms diff --git a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp similarity index 75% rename from modules/qds/include/opencv2/qds/quasiDenseStereo.hpp rename to modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index 12a5fb485d2..08e7fe0a21b 100644 --- a/modules/qds/include/opencv2/qds/quasiDenseStereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -51,9 +51,9 @@ the use of this software, even if advised of the possibility of such damage. namespace cv { -namespace qds +namespace stereo { -/** \addtogroup qds +/** \addtogroup stereo * @{ */ @@ -104,6 +104,27 @@ typedef std::priority_queue, std::less > t_matc /** * @brief Class containing the methods needed for Quasi Dense Stereo computation. * + * This module contains the code to perform quasi dense stereo matching. + * The method initially starts with a sparse 3D reconstruction based on feature matching across a + * stereo image pair and subsequently propagates the structure into neighboring image regions. + * To obtain initial seed correspondences, the algorithm locates Shi and Tomashi features in the + * left image of the stereo pair and then tracks them using pyramidal Lucas-Kanade in the right image. + * To densify the sparse correspondences, the algorithm computes the zero-mean normalized + * cross-correlation (ZNCC) in small patches around every seed pair and uses it as a quality metric + * for each match. In this code, we introduce a custom structure to store the location and ZNCC value + * of correspondences called "Match". Seed Matches are stored in a priority queue sorted according to + * their ZNCC value, allowing for the best quality Match to be readily available. The algorithm pops + * Matches and uses them to extract new matches around them. This is done by considering a small + * neighboring area around each Seed and retrieving correspondences above a certain texture threshold + * that are not previously computed. New matches are stored in the seed priority queue and used as seeds. + * The propagation process ends when no additional matches can be retrieved. + * + * + * @sa This code represents the work presented in @cite Stoyanov2010. + * If this code is useful for your work please cite @cite Stoyanov2010. + * + * Also the original growing scheme idea is described in @cite Lhuillier2000 + * */ class CV_EXPORTS_W QuasiDenseStereo @@ -119,8 +140,7 @@ class CV_EXPORTS_W QuasiDenseStereo /** * @brief Load a file containing the configuration parameters of the class. * @param[in] filepath The location of the .YAML file containing the configuration parameters. - * @note default value is an empty string in which case the default - * parameters, specified in the qds/defaults.h header-file, are loaded. + * @note default value is an empty string in which case the default parameters will be loaded. * @retval 1: If the path is not empty and the program loaded the parameters successfully. * @retval 0: If the path is empty and the program loaded default parameters. * @retval -1: If the file location is not valid or the program could not open the file and @@ -130,7 +150,7 @@ class CV_EXPORTS_W QuasiDenseStereo * in case of video processing. * @sa loadParameters */ - CV_WRAP virtual int loadParameters(cv::String filepath="") = 0; + CV_EXPORTS virtual int loadParameters(cv::String filepath="") = 0; /** @@ -141,7 +161,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note This method can be used to generate a template file for tuning the class. * @sa loadParameters */ - CV_WRAP virtual int saveParameters(cv::String filepath="./qds_parameters.yaml") = 0; + CV_EXPORTS virtual int saveParameters(cv::String filepath="./qds_parameters.yaml") = 0; /** @@ -150,7 +170,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note The method clears the sMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - CV_WRAP virtual void getSparseMatches(std::vector &sMatches) = 0; + CV_EXPORTS virtual void getSparseMatches(std::vector &sMatches) = 0; /** @@ -159,7 +179,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note The method clears the dMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - CV_WRAP virtual void getDenseMatches(std::vector &dMatches) = 0; + CV_EXPORTS virtual void getDenseMatches(std::vector &dMatches) = 0; @@ -175,7 +195,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @sa sparseMatching * @sa quasiDenseMatching */ - CV_WRAP virtual void process(const cv::Mat &imgLeft ,const cv::Mat &imgRight) = 0; + CV_EXPORTS virtual void process(const cv::Mat &imgLeft ,const cv::Mat &imgRight) = 0; /** @@ -186,7 +206,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @retval cv::Point(0, 0) (NO_MATCH) if no match is found in the right image for the specified pixel location in the left image. * @note This method should be always called after process, otherwise the matches will not be correct. */ - CV_WRAP virtual cv::Point2f getMatch(const int x, const int y) = 0; + CV_EXPORTS virtual cv::Point2f getMatch(const int x, const int y) = 0; /** @@ -197,17 +217,17 @@ class CV_EXPORTS_W QuasiDenseStereo * @sa computeDisparity * @sa quantizeDisparity */ - CV_WRAP virtual cv::Mat getDisparity(uint8_t disparityLvls=50) = 0; + CV_EXPORTS virtual cv::Mat getDisparity(uint8_t disparityLvls=50) = 0; - CV_WRAP static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath =""); + CV_EXPORTS static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath =""); CV_PROP_RW PropagationParameters Param; }; } //namespace cv -} //namespace qds +} //namespace stereo /** @}*/ diff --git a/modules/qds/samples/dense_disparity.cpp b/modules/stereo/samples/dense_disparity.cpp similarity index 89% rename from modules/qds/samples/dense_disparity.cpp rename to modules/stereo/samples/dense_disparity.cpp index b2284513dc0..cff0264a6bf 100644 --- a/modules/qds/samples/dense_disparity.cpp +++ b/modules/stereo/samples/dense_disparity.cpp @@ -1,7 +1,7 @@ #include #include #include -#include +#include @@ -20,7 +20,7 @@ int main() rightImg = imread("./imgRight.png", IMREAD_COLOR); cv::Size frameSize = leftImg.size(); // Initialize qds and start process. - Ptr stereo = qds::QuasiDenseStereo::create(frameSize); + Ptr stereo = stereo::QuasiDenseStereo::create(frameSize); uint8_t displvl = 80; // Number of disparity levels cv::Mat disp; @@ -31,7 +31,7 @@ int main() // Compute disparity between left and right channel of current frame. disp = stereo->getDisparity(displvl); - vector matches; + vector matches; stereo->getDenseMatches(matches); // Create three windows and show images. diff --git a/modules/qds/samples/export_param_file.cpp b/modules/stereo/samples/export_param_file.cpp similarity index 70% rename from modules/qds/samples/export_param_file.cpp rename to modules/stereo/samples/export_param_file.cpp index 96f41f9efca..a53a5b2c59b 100644 --- a/modules/qds/samples/export_param_file.cpp +++ b/modules/stereo/samples/export_param_file.cpp @@ -1,5 +1,5 @@ #include -#include +#include using namespace cv; using namespace std; @@ -11,7 +11,7 @@ int main(int argc, char* argv[]) parameterFileLocation = argv[1]; - Ptr stereo = qds::QuasiDenseStereo::create(cv::Size(5,5)); + Ptr stereo = stereo::QuasiDenseStereo::create(cv::Size(5,5)); stereo->saveParameters(parameterFileLocation); return 0; diff --git a/modules/qds/src/quasiDenseStereo.cpp b/modules/stereo/src/quasi_dense_stereo.cpp similarity index 98% rename from modules/qds/src/quasiDenseStereo.cpp rename to modules/stereo/src/quasi_dense_stereo.cpp index d05da5f9390..9b5dd44e189 100644 --- a/modules/qds/src/quasiDenseStereo.cpp +++ b/modules/stereo/src/quasi_dense_stereo.cpp @@ -19,7 +19,7 @@ are permitted provided that the following conditions are met: * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. + and/or other mstereorials provided with the distribution. * Neither the names of the copyright holders nor the names of the contributors may be used to endorse or promote products derived from this software @@ -38,9 +38,11 @@ the use of this software, even if advised of the possibility of such damage. */ #include "precomp.hpp" +#include +#include namespace cv { -namespace qds { +namespace stereo { #define NO_MATCH cv::Point(0,0) @@ -519,7 +521,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo //------------------------------------------------------------------------- - void getSparseMatches(std::vector &sMatches) override + void getSparseMatches(std::vector &sMatches) override { Match tmpMatch; sMatches.clear(); @@ -620,7 +622,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo return -1; } - void getDenseMatches(std::vector &dMatches) override + void getDenseMatches(std::vector &dMatches) override { Match tmpMatch; dMatches.clear(); diff --git a/modules/qds/tutorials/export_param_file.markdown b/modules/stereo/tutorials/export_param_file.markdown similarity index 90% rename from modules/qds/tutorials/export_param_file.markdown rename to modules/stereo/tutorials/export_param_file.markdown index cd202b0e714..4452c323f77 100644 --- a/modules/qds/tutorials/export_param_file.markdown +++ b/modules/stereo/tutorials/export_param_file.markdown @@ -18,7 +18,7 @@ This is very useful for fine-tuning the class' parameters on the fly. To extract parameter file you run the following code. ``` std::string parameterFileLocation = "./parameters.yaml"; -Ptr stereo = qds::QuasiDenseStereo::create(cv::Size(5,5)); +Ptr stereo = stereo::QuasiDenseStereo::create(cv::Size(5,5)); stereo->saveParameters(parameterFileLocation); ``` We make an instance of a `QuasiDenseStereo` object. Not specifying the second argument of the constructor, diff --git a/modules/qds/tutorials/quasi_dense_stereo.markdown b/modules/stereo/tutorials/quasi_dense_stereo.markdown similarity index 93% rename from modules/qds/tutorials/quasi_dense_stereo.markdown rename to modules/stereo/tutorials/quasi_dense_stereo.markdown index abcf6091350..ae60d664079 100644 --- a/modules/qds/tutorials/quasi_dense_stereo.markdown +++ b/modules/stereo/tutorials/quasi_dense_stereo.markdown @@ -26,7 +26,7 @@ rightImg = imread("./imgRight.png", IMREAD_COLOR); We need to know the frame size of a single image, in order to create an instance of a `QuasiDesnseStereo` object. ``` cv::Size frameSize = leftImg.size(); -Ptr stereo = qds::QuasiDenseStereo::create(frameSize); +Ptr stereo = stereo::QuasiDenseStereo::create(frameSize); ``` Because we didn't specify the second argument in the constructor, the `QuasiDesnseStereo` object will load default parameters. @@ -50,6 +50,6 @@ cv::imshow("disparity map", disp); At this point we can also extract all the corresponding points using `getDenseMatches()` method. ``` -vector matches; +vector matches; stereo->getDenseMatches(matches); ``` From 2bb9994a6120843c84895109d20d4aed7a8f4302 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Tue, 18 Dec 2018 10:06:35 +0000 Subject: [PATCH 18/29] Remove CV_PROP_RW flag. --- .../opencv2/stereo/quasi_dense_stereo.hpp | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index 08e7fe0a21b..634d808d8fd 100644 --- a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -61,9 +61,9 @@ namespace stereo // A basic match structure struct CV_EXPORTS_W_SIMPLE Match { - CV_PROP_RW cv::Point2i p0; - CV_PROP_RW cv::Point2i p1; - CV_PROP_RW float corr; + cv::Point2i p0; + cv::Point2i p1; + float corr; bool operator < (const Match & rhs) const//fixme may be used uninitialized in this function { @@ -72,29 +72,29 @@ struct CV_EXPORTS_W_SIMPLE Match }; struct CV_EXPORTS_W_SIMPLE PropagationParameters { - CV_PROP_RW int corrWinSizeX; // similarity window - CV_PROP_RW int corrWinSizeY; + int corrWinSizeX; // similarity window + int corrWinSizeY; - CV_PROP_RW int borderX; // border to ignore - CV_PROP_RW int borderY; + int borderX; // border to ignore + int borderY; //matching - CV_PROP_RW float correlationThreshold; // correlation threshold - CV_PROP_RW float textrureThreshold; // texture threshold + float correlationThreshold; // correlation threshold + float textrureThreshold; // texture threshold - CV_PROP_RW int neighborhoodSize; // neighborhood size - CV_PROP_RW int disparityGradient; // disparity gradient threshold + int neighborhoodSize; // neighborhood size + int disparityGradient; // disparity gradient threshold // Parameters for LK flow algorithm - CV_PROP_RW int lkTemplateSize; - CV_PROP_RW int lkPyrLvl; - CV_PROP_RW int lkTermParam1; - CV_PROP_RW float lkTermParam2; + int lkTemplateSize; + int lkPyrLvl; + int lkTermParam1; + float lkTermParam2; // Parameters for GFT algorithm. - CV_PROP_RW float gftQualityThres; - CV_PROP_RW int gftMinSeperationDist; - CV_PROP_RW int gftMaxNumFeatures; + float gftQualityThres; + int gftMinSeperationDist; + int gftMaxNumFeatures; }; @@ -223,7 +223,7 @@ class CV_EXPORTS_W QuasiDenseStereo CV_EXPORTS static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath =""); - CV_PROP_RW PropagationParameters Param; + PropagationParameters Param; }; } //namespace cv From e2963f0e02c173cc5b1163123e2da071c7f27c95 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Tue, 18 Dec 2018 10:53:45 +0000 Subject: [PATCH 19/29] Remove CV_EXPORTS flags from class members. --- .../opencv2/stereo/quasi_dense_stereo.hpp | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index 634d808d8fd..8958ff04ec8 100644 --- a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -1,4 +1,4 @@ -/* +CV_EXPORTS/* By downloading, copying, installing or using the software you agree to this license. If you do not agree to this license, do not download, install, copy or use the software. @@ -59,7 +59,7 @@ namespace stereo // A basic match structure -struct CV_EXPORTS_W_SIMPLE Match +struct CV_EXPORTS Match { cv::Point2i p0; cv::Point2i p1; @@ -70,7 +70,7 @@ struct CV_EXPORTS_W_SIMPLE Match return this->corr < rhs.corr; } }; -struct CV_EXPORTS_W_SIMPLE PropagationParameters +struct CV_EXPORTS PropagationParameters { int corrWinSizeX; // similarity window int corrWinSizeY; @@ -127,7 +127,7 @@ typedef std::priority_queue, std::less > t_matc * */ -class CV_EXPORTS_W QuasiDenseStereo +class CV_EXPORTS QuasiDenseStereo { public: /** @@ -150,7 +150,7 @@ class CV_EXPORTS_W QuasiDenseStereo * in case of video processing. * @sa loadParameters */ - CV_EXPORTS virtual int loadParameters(cv::String filepath="") = 0; + virtual int loadParameters(cv::String filepath="") = 0; /** @@ -161,7 +161,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note This method can be used to generate a template file for tuning the class. * @sa loadParameters */ - CV_EXPORTS virtual int saveParameters(cv::String filepath="./qds_parameters.yaml") = 0; + virtual int saveParameters(cv::String filepath="./qds_parameters.yaml") = 0; /** @@ -170,7 +170,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note The method clears the sMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - CV_EXPORTS virtual void getSparseMatches(std::vector &sMatches) = 0; + virtual void getSparseMatches(std::vector &sMatches) = 0; /** @@ -179,7 +179,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @note The method clears the dMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - CV_EXPORTS virtual void getDenseMatches(std::vector &dMatches) = 0; + virtual void getDenseMatches(std::vector &dMatches) = 0; @@ -195,7 +195,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @sa sparseMatching * @sa quasiDenseMatching */ - CV_EXPORTS virtual void process(const cv::Mat &imgLeft ,const cv::Mat &imgRight) = 0; + virtual void process(const cv::Mat &imgLeft ,const cv::Mat &imgRight) = 0; /** @@ -206,7 +206,7 @@ class CV_EXPORTS_W QuasiDenseStereo * @retval cv::Point(0, 0) (NO_MATCH) if no match is found in the right image for the specified pixel location in the left image. * @note This method should be always called after process, otherwise the matches will not be correct. */ - CV_EXPORTS virtual cv::Point2f getMatch(const int x, const int y) = 0; + virtual cv::Point2f getMatch(const int x, const int y) = 0; /** @@ -217,10 +217,10 @@ class CV_EXPORTS_W QuasiDenseStereo * @sa computeDisparity * @sa quantizeDisparity */ - CV_EXPORTS virtual cv::Mat getDisparity(uint8_t disparityLvls=50) = 0; + virtual cv::Mat getDisparity(uint8_t disparityLvls=50) = 0; - CV_EXPORTS static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath =""); + static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath =""); PropagationParameters Param; From 4d04ae7adf3c1561381913ea87addbf982210980 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Tue, 18 Dec 2018 12:21:33 +0000 Subject: [PATCH 20/29] Fix: Removed misplaced flag --- modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index 8958ff04ec8..223548a2745 100644 --- a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -1,4 +1,4 @@ -CV_EXPORTS/* +/* By downloading, copying, installing or using the software you agree to this license. If you do not agree to this license, do not download, install, copy or use the software. @@ -170,7 +170,7 @@ class CV_EXPORTS QuasiDenseStereo * @note The method clears the sMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - virtual void getSparseMatches(std::vector &sMatches) = 0; + virtual void getSparseMatches(std::vector &sMatches) = 0; /** From 2c3b4805305028fe8b3b434c1b0ca4b1b8a425c5 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 19 Dec 2018 14:39:00 +0000 Subject: [PATCH 21/29] Remove empty lines. --- modules/stereo/doc/stereo.bib | 3 --- 1 file changed, 3 deletions(-) diff --git a/modules/stereo/doc/stereo.bib b/modules/stereo/doc/stereo.bib index b776036f207..353e6d1b71a 100644 --- a/modules/stereo/doc/stereo.bib +++ b/modules/stereo/doc/stereo.bib @@ -17,9 +17,6 @@ @InProceedings{Stoyanov2010 isbn="978-3-642-15705-9" } - - - @article{Lhuillier2000, abstract = {A new robust dense matching algorithm is introduced. The algorithm$\backslash$nstarts from matching the most textured points, then a match propagation$\backslash$nalgorithm is developed with the best first strategy to dense matching.$\backslash$nNext, the matching map is regularised by using the local geometric$\backslash$nconstraints encoded by planar affine applications and by using the$\backslash$nglobal geometric constraint encoded by the fundamental matrix. Two most$\backslash$ndistinctive features are a match propagation strategy developed by$\backslash$nanalogy to region growing and a successive regularisation by local and$\backslash$nglobal geometric constraints. The algorithm is efficient, robust and can$\backslash$ncope with wide disparity. The algorithm is demonstrated on many real$\backslash$nimage pairs, and applications on image interpolation and a creation of$\backslash$nnovel views are also presented}, author = {Lhuillier, Maxime and Quan, Long}, From e2cdd27dc4e4214361cd5638fed1d42f029d93a2 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 19 Dec 2018 14:44:46 +0000 Subject: [PATCH 22/29] Move queue to private headers. --- modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp | 4 ---- modules/stereo/src/quasi_dense_stereo.cpp | 4 ++++ 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index 223548a2745..0c524684249 100644 --- a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -45,8 +45,6 @@ the use of this software, even if advised of the possibility of such damage. #include -#include - namespace cv @@ -98,8 +96,6 @@ struct CV_EXPORTS PropagationParameters }; -typedef std::priority_queue, std::less > t_matchPriorityQueue; - /** * @brief Class containing the methods needed for Quasi Dense Stereo computation. diff --git a/modules/stereo/src/quasi_dense_stereo.cpp b/modules/stereo/src/quasi_dense_stereo.cpp index 9b5dd44e189..7dd6f62dfe3 100644 --- a/modules/stereo/src/quasi_dense_stereo.cpp +++ b/modules/stereo/src/quasi_dense_stereo.cpp @@ -40,12 +40,16 @@ the use of this software, even if advised of the possibility of such damage. #include "precomp.hpp" #include #include +#include + namespace cv { namespace stereo { #define NO_MATCH cv::Point(0,0) +typedef std::priority_queue, std::less > t_matchPriorityQueue; + class QuasiDenseStereoImpl : public QuasiDenseStereo { From a7d0556ef721cae541c0f7d867d227105f02dcb1 Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 19 Dec 2018 15:06:47 +0000 Subject: [PATCH 23/29] Fix default arguments of public methods. --- .../stereo/include/opencv2/stereo/quasi_dense_stereo.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index 0c524684249..957a4cad52e 100644 --- a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -146,7 +146,7 @@ class CV_EXPORTS QuasiDenseStereo * in case of video processing. * @sa loadParameters */ - virtual int loadParameters(cv::String filepath="") = 0; + virtual int loadParameters(cv::String filepath) = 0; /** @@ -157,7 +157,7 @@ class CV_EXPORTS QuasiDenseStereo * @note This method can be used to generate a template file for tuning the class. * @sa loadParameters */ - virtual int saveParameters(cv::String filepath="./qds_parameters.yaml") = 0; + virtual int saveParameters(cv::String filepath) = 0; /** @@ -216,7 +216,7 @@ class CV_EXPORTS QuasiDenseStereo virtual cv::Mat getDisparity(uint8_t disparityLvls=50) = 0; - static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath =""); + static cv::Ptr create(cv::Size monoImgSize, cv::String paramFilepath = cv::String()); PropagationParameters Param; From 2d8d4caa082490836e47f14892b43be2a3df91ea Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 19 Dec 2018 16:09:38 +0000 Subject: [PATCH 24/29] Add authors information and switch to the compact version of license header. --- .../opencv2/stereo/quasi_dense_stereo.hpp | 43 +++---------------- 1 file changed, 5 insertions(+), 38 deletions(-) diff --git a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index 957a4cad52e..49e554afe45 100644 --- a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -1,41 +1,8 @@ -/* -By downloading, copying, installing or using the software you agree to this license. -If you do not agree to this license, do not download, install, -copy or use the software. - - - License Agreement - For Open Source Computer Vision Library - (3-clause BSD License) - -Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. -Third party copyrights are property of their respective owners. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - - * Neither the names of the copyright holders nor the names of the contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -This software is provided by the copyright holders and contributors "as is" and -any express or implied warranties, including, but not limited to, the implied -warranties of merchantability and fitness for a particular purpose are disclaimed. -In no event shall copyright holders or contributors be liable for any direct, -indirect, incidental, special, exemplary, or consequential damages -(including, but not limited to, procurement of substitute goods or services; -loss of use, data, or profits; or business interruption) however caused -and on any theory of liability, whether in contract, strict liability, -or tort (including negligence or otherwise) arising in any way out of -the use of this software, even if advised of the possibility of such damage. - */ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. + +//authors: Danail Stoyanov, Evangelos Mazomenos, Dimitrios Psychogyios //__OPENCV_QUASI_DENSE_STEREO_H__ From bb769ad16f5a1415979feabaa6affc32d14d900c Mon Sep 17 00:00:00 2001 From: Dimitrios Date: Wed, 19 Dec 2018 18:18:07 +0000 Subject: [PATCH 25/29] Reorganize and fix markdown files. Create a table of content and move tutorials in new directories. Modify samples and tutorials to use snippet and include Doxygen commands. --- modules/stereo/samples/dense_disparity.cpp | 36 +++++++++++-------- modules/stereo/samples/export_param_file.cpp | 10 ++++-- .../qds_export_parameters.markdown} | 15 ++++---- .../qds_quasi_dense_stereo.markdown} | 35 ++++++------------ ...ble_of_content_quasi_dense_stereo.markdown | 14 ++++++++ 5 files changed, 59 insertions(+), 51 deletions(-) rename modules/stereo/tutorials/{export_param_file.markdown => qds_export_parameters/qds_export_parameters.markdown} (56%) rename modules/stereo/tutorials/{quasi_dense_stereo.markdown => qds_quasi_dense_stereo/qds_quasi_dense_stereo.markdown} (63%) create mode 100644 modules/stereo/tutorials/table_of_content_quasi_dense_stereo.markdown diff --git a/modules/stereo/samples/dense_disparity.cpp b/modules/stereo/samples/dense_disparity.cpp index cff0264a6bf..0e257def230 100644 --- a/modules/stereo/samples/dense_disparity.cpp +++ b/modules/stereo/samples/dense_disparity.cpp @@ -12,43 +12,49 @@ using namespace std; int main() { - - Mat rightImg, leftImg; - - //Read video meta-data to determine the correct frame size for initialization. +//! [load] + cv::Mat rightImg, leftImg; leftImg = imread("./imgLeft.png", IMREAD_COLOR); rightImg = imread("./imgRight.png", IMREAD_COLOR); +//! [load] + + +//! [create] cv::Size frameSize = leftImg.size(); - // Initialize qds and start process. Ptr stereo = stereo::QuasiDenseStereo::create(frameSize); +//! [create] - uint8_t displvl = 80; // Number of disparity levels - cv::Mat disp; - // Compute dense stereo. +//! [process] stereo->process(leftImg, rightImg); +//! [process] - // Compute disparity between left and right channel of current frame. + +//! [disp] + uint8_t displvl = 80; + cv::Mat disp; disp = stereo->getDisparity(displvl); + cv::namedWindow("disparity map"); + cv::imshow("disparity map", disp); +//! [disp] - vector matches; - stereo->getDenseMatches(matches); - // Create three windows and show images. cv::namedWindow("right channel"); cv::namedWindow("left channel"); - cv::namedWindow("disparity map"); - cv::imshow("disparity map", disp); cv::imshow("left channel", leftImg); cv::imshow("right channel", rightImg); - std::ofstream dense("./dense.txt", std::ios::out); +//! [export] + vector matches; + stereo->getDenseMatches(matches); + std::ofstream dense("./dense.txt", std::ios::out); for (uint i=0; i< matches.size(); i++) { dense << matches[i].p0 << matches[i].p1 << endl; } dense.close(); +//! [export] diff --git a/modules/stereo/samples/export_param_file.cpp b/modules/stereo/samples/export_param_file.cpp index a53a5b2c59b..9ea90002e5f 100644 --- a/modules/stereo/samples/export_param_file.cpp +++ b/modules/stereo/samples/export_param_file.cpp @@ -6,13 +6,17 @@ using namespace std; int main(int argc, char* argv[]) { +//! [create] + Ptr stereo = stereo::QuasiDenseStereo::create(cv::Size(5,5)); +//! [create] + + +//! [write] std::string parameterFileLocation = "./parameters.yaml"; if (argc > 1) parameterFileLocation = argv[1]; - - - Ptr stereo = stereo::QuasiDenseStereo::create(cv::Size(5,5)); stereo->saveParameters(parameterFileLocation); +//! [write] return 0; } diff --git a/modules/stereo/tutorials/export_param_file.markdown b/modules/stereo/tutorials/qds_export_parameters/qds_export_parameters.markdown similarity index 56% rename from modules/stereo/tutorials/export_param_file.markdown rename to modules/stereo/tutorials/qds_export_parameters/qds_export_parameters.markdown index 4452c323f77..f4f6e3af594 100644 --- a/modules/stereo/tutorials/export_param_file.markdown +++ b/modules/stereo/tutorials/qds_export_parameters/qds_export_parameters.markdown @@ -1,4 +1,4 @@ -Exporting a template parameter file {#export_param_file} +Exporting a template parameter file {#tutorial_qds_export_parameters} ================== Goal @@ -8,19 +8,16 @@ In this tutorial you will learn how to - create a simple parameter file template. ------------ -[Source Code](../samples/export_param_file.cpp) +@include ./samples/export_param_file.cpp ## Explanation: The class supports loading configuration parameters from a .yaml file using the method `loadParameters()`. This is very useful for fine-tuning the class' parameters on the fly. To extract a template of this parameter file you run the following code. -``` -std::string parameterFileLocation = "./parameters.yaml"; -Ptr stereo = stereo::QuasiDenseStereo::create(cv::Size(5,5)); -stereo->saveParameters(parameterFileLocation); -``` -We make an instance of a `QuasiDenseStereo` object. Not specifying the second argument of the constructor, + +We create an instance of a `QuasiDenseStereo` object. Not specifying the second argument of the constructor, makes the object to load default parameters. +@snippet ./samples/export_param_file.cpp create By calling the method `saveParameters()`, we store the template file to the location specified by `parameterFileLocation` +@snippet ./samples/export_param_file.cpp write diff --git a/modules/stereo/tutorials/quasi_dense_stereo.markdown b/modules/stereo/tutorials/qds_quasi_dense_stereo/qds_quasi_dense_stereo.markdown similarity index 63% rename from modules/stereo/tutorials/quasi_dense_stereo.markdown rename to modules/stereo/tutorials/qds_quasi_dense_stereo/qds_quasi_dense_stereo.markdown index ae60d664079..fce1e97493c 100644 --- a/modules/stereo/tutorials/quasi_dense_stereo.markdown +++ b/modules/stereo/tutorials/qds_quasi_dense_stereo/qds_quasi_dense_stereo.markdown @@ -1,4 +1,4 @@ -Quasi dense Stereo {#quasi_dense_stereo} +Quasi dense Stereo {#tutorial_qds_quasi_dense_stereo} ================== Goal @@ -9,8 +9,7 @@ In this tutorial you will learn how to - Configure a QuasiDenseStero object - Compute dense Stereo correspondences. ------------ -[Source Code](../samples/dense_disparity.cpp) +@include ./samples/dense_disparity.cpp ## Explanation: @@ -18,19 +17,15 @@ The program loads a stereo image pair. After importing the images. -``` -Mat rightImg, leftImg; -leftImg = imread("./imgLeft.png", IMREAD_COLOR); -rightImg = imread("./imgRight.png", IMREAD_COLOR); -``` +@snippet ./samples/dense_disparity.cpp load We need to know the frame size of a single image, in order to create an instance of a `QuasiDesnseStereo` object. -``` -cv::Size frameSize = leftImg.size(); -Ptr stereo = stereo::QuasiDenseStereo::create(frameSize); -``` + +@snippet ./samples/dense_disparity.cpp create + Because we didn't specify the second argument in the constructor, the `QuasiDesnseStereo` object will load default parameters. +@snippet ./samples/dense_disparity.cpp process We can then pass the imported stereo images in the process method like this ``` stereo->process(leftImg, rightImg); @@ -40,16 +35,8 @@ The process method contains most of the functionality of the class and does two - Based on those sparse stereo points, densifies the stereo correspondences using Quasi Dense Stereo method. After the execution of `process()` we can display the disparity Image of the stereo. -``` -int displvl = 80; -Mat disp; -disp = stereo->getDisparity(displvl); -cv::namedWindow("disparity map"); -cv::imshow("disparity map", disp); -``` +@snippet ./samples/dense_disparity.cpp disp -At this point we can also extract all the corresponding points using `getDenseMatches()` method. -``` -vector matches; -stereo->getDenseMatches(matches); -``` + +At this point we can also extract all the corresponding points using `getDenseMatches()` method and export them in a file. +@snippet ./samples/dense_disparity.cpp export diff --git a/modules/stereo/tutorials/table_of_content_quasi_dense_stereo.markdown b/modules/stereo/tutorials/table_of_content_quasi_dense_stereo.markdown new file mode 100644 index 00000000000..7da5b186d45 --- /dev/null +++ b/modules/stereo/tutorials/table_of_content_quasi_dense_stereo.markdown @@ -0,0 +1,14 @@ +Quasi Dense Stereo (stereo module) {#tutorial_table_of_content_quasi_dense_stereo} +========================================================== + +Quasi Dense Stereo is method for performing dense stereo matching. `QuasiDenseStereo` implements this process. +The code uses pyramidal Lucas-Kanade with Shi-Tomasi features to get the initial seed correspondences. +Then these seeds are propagated by using mentioned growing scheme. + +- @subpage tutorial_qds_quasi_dense_stereo + + Example showing how to get dense correspondences from a stereo image pair. + +- @subpage tutorial_qds_export_parameters + + Example showing how to genereate a parameter file template. From 326d6d7f46ff39cf15fd804b1b8efe2cd40e19d5 Mon Sep 17 00:00:00 2001 From: Dimitrios Psychogyios Date: Wed, 19 Dec 2018 18:37:44 +0000 Subject: [PATCH 26/29] Change argument name dMatch->denseMatch, to avoid confusion with cv::DMatch build-in type. --- .../stereo/include/opencv2/stereo/quasi_dense_stereo.hpp | 6 +++--- modules/stereo/src/quasi_dense_stereo.cpp | 8 +++----- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp index 49e554afe45..b302c133464 100644 --- a/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp +++ b/modules/stereo/include/opencv2/stereo/quasi_dense_stereo.hpp @@ -138,11 +138,11 @@ class CV_EXPORTS QuasiDenseStereo /** * @brief Get The dense corresponding points. - * @param[out] dMatches A vector containing all dense matches. - * @note The method clears the dMatches vector. + * @param[out] denseMatches A vector containing all dense matches. + * @note The method clears the denseMatches vector. * @note The returned Match elements inside the sMatches vector, do not use corr member. */ - virtual void getDenseMatches(std::vector &dMatches) = 0; + virtual void getDenseMatches(std::vector &denseMatches) = 0; diff --git a/modules/stereo/src/quasi_dense_stereo.cpp b/modules/stereo/src/quasi_dense_stereo.cpp index 7dd6f62dfe3..3ab49c83345 100644 --- a/modules/stereo/src/quasi_dense_stereo.cpp +++ b/modules/stereo/src/quasi_dense_stereo.cpp @@ -626,11 +626,10 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo return -1; } - void getDenseMatches(std::vector &dMatches) override + void getDenseMatches(std::vector &denseMatches) override { Match tmpMatch; - dMatches.clear(); - // dMatches.resize(dMatchesLen); + denseMatches.clear(); for (int row=0; row Date: Wed, 19 Dec 2018 18:53:22 +0000 Subject: [PATCH 27/29] Remove duplicate snippet. --- .../qds_quasi_dense_stereo/qds_quasi_dense_stereo.markdown | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/modules/stereo/tutorials/qds_quasi_dense_stereo/qds_quasi_dense_stereo.markdown b/modules/stereo/tutorials/qds_quasi_dense_stereo/qds_quasi_dense_stereo.markdown index fce1e97493c..34c1bb6480d 100644 --- a/modules/stereo/tutorials/qds_quasi_dense_stereo/qds_quasi_dense_stereo.markdown +++ b/modules/stereo/tutorials/qds_quasi_dense_stereo/qds_quasi_dense_stereo.markdown @@ -15,7 +15,6 @@ In this tutorial you will learn how to The program loads a stereo image pair. - After importing the images. @snippet ./samples/dense_disparity.cpp load We need to know the frame size of a single image, in order to create an instance of a `QuasiDesnseStereo` object. @@ -25,11 +24,9 @@ We need to know the frame size of a single image, in order to create an instance Because we didn't specify the second argument in the constructor, the `QuasiDesnseStereo` object will load default parameters. -@snippet ./samples/dense_disparity.cpp process We can then pass the imported stereo images in the process method like this -``` -stereo->process(leftImg, rightImg); -``` +@snippet ./samples/dense_disparity.cpp process + The process method contains most of the functionality of the class and does two main things. - Computes a sparse stereo based in "Good Features to Track" and "pyramidal Lucas-Kanade" flow algorithm - Based on those sparse stereo points, densifies the stereo correspondences using Quasi Dense Stereo method. From 045ad892ca938d19b5056de8f4d17bfeee2fa3ea Mon Sep 17 00:00:00 2001 From: Dimitrios Psychogyios Date: Thu, 20 Dec 2018 14:20:22 +0000 Subject: [PATCH 28/29] Fix: change vector resize to reserve. --- modules/stereo/src/quasi_dense_stereo.cpp | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/modules/stereo/src/quasi_dense_stereo.cpp b/modules/stereo/src/quasi_dense_stereo.cpp index 3ab49c83345..6b29784edfc 100644 --- a/modules/stereo/src/quasi_dense_stereo.cpp +++ b/modules/stereo/src/quasi_dense_stereo.cpp @@ -176,6 +176,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo void quasiDenseMatching(const std::vector< cv::Point2f > &featuresLeft, const std::vector< cv::Point2f > &featuresRight) { + dMatchesLen = 0; refMap = cv::Mat_(cv::Size(width, height), cv::Point2i(0, 0)); mtcMap = cv::Point2i(0, 0); @@ -269,6 +270,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo // Unique match refMap.at(lm.p0.y, lm.p0.x) = lm.p1; mtcMap.at(lm.p1.y, lm.p1.x) = lm.p0; + dMatchesLen++; // Add to the seed list seeds.push(lm); } @@ -529,7 +531,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo { Match tmpMatch; sMatches.clear(); - sMatches.resize(leftFeatures.size()); + sMatches.reserve(leftFeatures.size()); for (uint i=0; i(y, x); } - cv::Mat getDisparity(uint8_t disparityLvls) override { computeDisparity(refMap, disparity); @@ -683,6 +685,7 @@ class QuasiDenseStereoImpl : public QuasiDenseStereo // Width and height of a single image. int width; int height; + int dMatchesLen; // Containers to store input images. cv::Mat grayLeft; cv::Mat grayRight; From 4a291081a9faeb3fb607f86674deb732ce42c01b Mon Sep 17 00:00:00 2001 From: Dimitrios Psychogyios Date: Thu, 27 Dec 2018 17:19:49 +0000 Subject: [PATCH 29/29] Fix: replace extensive license header with the compact version. --- modules/stereo/src/quasi_dense_stereo.cpp | 41 ++--------------------- 1 file changed, 3 insertions(+), 38 deletions(-) diff --git a/modules/stereo/src/quasi_dense_stereo.cpp b/modules/stereo/src/quasi_dense_stereo.cpp index 6b29784edfc..7a6513cb843 100644 --- a/modules/stereo/src/quasi_dense_stereo.cpp +++ b/modules/stereo/src/quasi_dense_stereo.cpp @@ -1,41 +1,6 @@ -/* -By downloading, copying, installing or using the software you agree to this license. -If you do not agree to this license, do not download, install, -copy or use the software. - - - License Agreement - For Open Source Computer Vision Library - (3-clause BSD License) - -Copyright (C) 2015-2018, OpenCV Foundation, all rights reserved. -Third party copyrights are property of their respective owners. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - * Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other mstereorials provided with the distribution. - - * Neither the names of the copyright holders nor the names of the contributors - may be used to endorse or promote products derived from this software - without specific prior written permission. - -This software is provided by the copyright holders and contributors "as is" and -any express or implied warranties, including, but not limited to, the implied -warranties of merchantability and fitness for a particular purpose are disclaimed. -In no event shall copyright holders or contributors be liable for any direct, -indirect, incidental, special, exemplary, or consequential damages -(including, but not limited to, procurement of substitute goods or services; -loss of use, data, or profits; or business interruption) however caused -and on any theory of liability, whether in contract, strict liability, -or tort (including negligence or otherwise) arising in any way out of -the use of this software, even if advised of the possibility of such damage. - */ +// This file is part of OpenCV project. +// It is subject to the license terms in the LICENSE file found in the top-level directory +// of this distribution and at http://opencv.org/license.html. #include "precomp.hpp" #include