Merge pull request #3351 from bit1002lst:decode_multi_qrcodes

asmorkalov · web-flow · commit cef97245bdbb · 2022-09-28T12:49:02.000+03:00
fix the bug that cannot detect multi qrcode when use_nn_detector is false
diff --git a/modules/wechat_qrcode/samples/qrcode_example_without_nn.cpp b/modules/wechat_qrcode/samples/qrcode_example_without_nn.cpp
@@ -0,0 +1,67 @@
+#include <iostream>
+#include <opencv2/core.hpp>
+#include <opencv2/highgui.hpp>
+#include <opencv2/imgproc.hpp>
+
+using namespace std;
+using namespace cv;
+
+#include <opencv2/wechat_qrcode.hpp>
+
+int main(int argc, char* argv[]) {
+    cout << endl << argv[0] << endl << endl;
+    cout << "A demo program of WeChat QRCode Detector: " << endl;
+
+    Mat img;
+    int camIdx = -1;
+    if (argc > 1) {
+        bool live = strcmp(argv[1], "-camera") == 0;
+        if (live) {
+            camIdx = argc > 2 ? atoi(argv[2]) : 0;
+        } else {
+            img = imread(argv[1]);
+        }
+    } else {
+        cout << "    Usage: " << argv[0] << " <input_image>" << endl;
+        return 0;
+    }
+    // The model is downloaded to ${CMAKE_BINARY_DIR}/downloads/wechat_qrcode if cmake runs without warnings,
+    // otherwise you can download them from https://github.com/WeChatCV/opencv_3rdparty/tree/wechat_qrcode.
+    Ptr<wechat_qrcode::WeChatQRCode> detector;
+
+    try {
+        detector = makePtr<wechat_qrcode::WeChatQRCode>("", "", "", "");
+    } catch (const std::exception& e) {
+        cout <<
+            "\n---------------------------------------------------------------\n"
+            "Failed to initialize WeChatQRCode.\n"
+            "---------------------------------------------------------------\n";
+        cout << e.what() << endl;
+        return 0;
+    }
+    string prevstr = "";
+    vector<Mat> points;
+
+    if (camIdx < 0) {
+        auto res = detector->detectAndDecode(img, points);
+        for (const auto& t : res) cout << t << endl;
+    } else {
+        VideoCapture cap(camIdx);
+        for(;;) {
+            cap >> img;
+            if (img.empty())
+                break;
+            auto res = detector->detectAndDecode(img, points);
+            for (const auto& t : res) {
+                if (t != prevstr)
+                    cout << t << endl;
+            }
+            if (!res.empty())
+                prevstr = res.back();
+            imshow("image", img);
+            if (waitKey(30) >= 0)
+                break;
+        }
+    }
+    return 0;
+}
diff --git a/modules/wechat_qrcode/samples/qrcode_without_nn.py b/modules/wechat_qrcode/samples/qrcode_without_nn.py
@@ -0,0 +1,50 @@
+import cv2
+import sys
+
+print(sys.argv[0])
+print('A demo program of WeChat QRCode Detector:')
+camIdx = -1
+if len(sys.argv) > 1:
+    if sys.argv[1] == "-camera":
+        camIdx = int(sys.argv[2]) if len(sys.argv)>2 else 0
+    img = cv2.imread(sys.argv[1])
+else:
+    print("    Usage: " + sys.argv[0] + "  <input_image>")
+    exit(0)
+
+# For python API generator, it follows the template: {module_name}_{class_name},
+# so it is a little weird.
+# The model is downloaded to ${CMAKE_BINARY_DIR}/downloads/wechat_qrcode if cmake runs without warnings,
+# otherwise you can download them from https://github.com/WeChatCV/opencv_3rdparty/tree/wechat_qrcode.
+try:
+    detector = cv2.wechat_qrcode_WeChatQRCode(
+        "", "", "", "")
+except:
+    print("---------------------------------------------------------------")
+    print("Failed to initialize WeChatQRCode.")
+    print("---------------------------------------------------------------")
+    exit(0)
+
+prevstr = ""
+
+if camIdx < 0:
+    res, points = detector.detectAndDecode(img)
+    print(res,points)
+else:
+    cap = cv2.VideoCapture(camIdx)
+    while True:
+        res, img = cap.read()
+        if img is None:
+            break
+        res, points = detector.detectAndDecode(img)
+        for t in res:
+            if t != prevstr:
+                print(t)
+        if res:
+            prevstr = res[-1]
+        cv2.imshow("image", img)
+        if cv2.waitKey(30) >= 0:
+            break
+    # When everything done, release the capture
+    cap.release()
+    cv2.destroyAllWindows()
diff --git a/modules/wechat_qrcode/src/decodermgr.cpp b/modules/wechat_qrcode/src/decodermgr.cpp
@@ -18,7 +18,7 @@ using zxing::Result;
 using zxing::UnicomBlock;
 namespace cv {
 namespace wechat_qrcode {
-int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, string& result, vector<Point2f>& points) {
+int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, vector<string>& results, vector<vector<Point2f>>& zxing_points) {
     int width = src.cols;
     int height = src.rows;
     if (width <= 20 || height <= 20)
@@ -28,7 +28,7 @@ int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, string& result, v
     zxing::ArrayRef<uint8_t> scaled_img_zx =
         zxing::ArrayRef<uint8_t>(new zxing::Array<uint8_t>(scaled_img_data));
 
-    zxing::Ref<zxing::Result> zx_result;
+    vector<zxing::Ref<zxing::Result>> zx_results;
 
     decode_hints_.setUseNNDetector(use_nn_detector);
 
@@ -43,16 +43,20 @@ int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, string& result, v
         } else {
             source->reset(scaled_img_zx.data(), width, height);
         }
-        int ret = TryDecode(source, zx_result);
+        int ret = TryDecode(source, zx_results);
         if (!ret) {
-            result = zx_result->getText()->getText();
-            auto result_points = zx_result->getResultPoints();
-            for(int i = 0; i < result_points->size() / 4; i++) {
-                const int ind = i * 4;
-                for (int j = 1; j < 4; j++)
-                    points.emplace_back(result_points[ind+j]->getX(), result_points[ind+j]->getY());
-
-               points.emplace_back(result_points[ind]->getX(), result_points[ind]->getY());
+            for(size_t k = 0; k < zx_results.size(); k++) {
+                results.emplace_back(zx_results[k]->getText()->getText());
+                vector<Point2f> tmp_qr_points;
+                auto tmp_zx_points = zx_results[k]->getResultPoints();
+                for (int i = 0; i < tmp_zx_points->size() / 4; i++) {
+                    const int ind = i * 4;
+                    for (int j = 1; j < 4; j++){
+                        tmp_qr_points.emplace_back(tmp_zx_points[ind + j]->getX(), tmp_zx_points[ind + j]->getY());
+                    }
+                    tmp_qr_points.emplace_back(tmp_zx_points[ind]->getX(), tmp_zx_points[ind]->getY());
+                }
+                zxing_points.push_back(tmp_qr_points);
             }
             return ret;
         }
@@ -62,7 +66,7 @@ int DecoderMgr::decodeImage(cv::Mat src, bool use_nn_detector, string& result, v
     return -1;
 }
 
-int DecoderMgr::TryDecode(Ref<LuminanceSource> source, Ref<Result>& result) {
+int DecoderMgr::TryDecode(Ref<LuminanceSource> source, vector<Ref<Result>>& results) {
     int res = -1;
     string cell_result;
 
@@ -71,17 +75,17 @@ int DecoderMgr::TryDecode(Ref<LuminanceSource> source, Ref<Result>& result) {
     zxing::Ref<zxing::BinaryBitmap> binary_bitmap(new BinaryBitmap(binarizer));
     binary_bitmap->m_poUnicomBlock = qbarUicomBlock_;
 
-    result = Decode(binary_bitmap, decode_hints_);
-    res = (result == NULL) ? 1 : 0;
+    results = Decode(binary_bitmap, decode_hints_);
+    res = (results.size() == 0) ? 1 : 0;
 
     if (res == 0) {
-        result->setBinaryMethod(int(binarizer_mgr_.GetCurBinarizer()));
+        results[0]->setBinaryMethod(int(binarizer_mgr_.GetCurBinarizer()));
     }
 
     return res;
 }
 
-Ref<Result> DecoderMgr::Decode(Ref<BinaryBitmap> image, DecodeHints hints) {
+vector<Ref<Result>> DecoderMgr::Decode(Ref<BinaryBitmap> image, DecodeHints hints) {
     return reader_->decode(image, hints);
 }
 }  // namespace wechat_qrcode
diff --git a/modules/wechat_qrcode/src/decodermgr.hpp b/modules/wechat_qrcode/src/decodermgr.hpp
@@ -26,7 +26,7 @@ class DecoderMgr {
     DecoderMgr() { reader_ = new zxing::qrcode::QRCodeReader(); };
     ~DecoderMgr(){};
 
-    int decodeImage(cv::Mat src, bool use_nn_detector, string& result, vector<Point2f>& points);
+    int decodeImage(cv::Mat src, bool use_nn_detector, vector<string>& result, vector<vector<Point2f>>& zxing_points);
 
 private:
     zxing::Ref<zxing::UnicomBlock> qbarUicomBlock_;
@@ -35,10 +35,10 @@ class DecoderMgr {
     zxing::Ref<zxing::qrcode::QRCodeReader> reader_;
     BinarizerMgr binarizer_mgr_;
 
-    zxing::Ref<zxing::Result> Decode(zxing::Ref<zxing::BinaryBitmap> image,
+    vector<zxing::Ref<zxing::Result>> Decode(zxing::Ref<zxing::BinaryBitmap> image,
                                      zxing::DecodeHints hints);
 
-    int TryDecode(zxing::Ref<zxing::LuminanceSource> source, zxing::Ref<zxing::Result>& result);
+    int TryDecode(zxing::Ref<zxing::LuminanceSource> source, vector<zxing::Ref<zxing::Result>>& result);
 };
 
 }  // namespace wechat_qrcode
diff --git a/modules/wechat_qrcode/src/wechat_qrcode.cpp b/modules/wechat_qrcode/src/wechat_qrcode.cpp
@@ -144,21 +144,44 @@ vector<string> WeChatQRCode::Impl::decode(const Mat& img, vector<Mat>& candidate
                 super_resolution_model_->processImageScale(cropped_img, cur_scale, use_nn_sr_);
             string result;
             DecoderMgr decodemgr;
-            vector<Point2f> points_qr;
-            auto ret = decodemgr.decodeImage(scaled_img, use_nn_detector_, result, points_qr);
+            vector<vector<Point2f>> zxing_points, check_points;
+            auto ret = decodemgr.decodeImage(scaled_img, use_nn_detector_, decode_results, zxing_points);
             if (ret == 0) {
-                for (auto&& pt: points_qr) {
-                    pt /= cur_scale;
+                for(size_t i = 0; i <zxing_points.size(); i++){
+                    vector<Point2f> points_qr = zxing_points[i];
+                    for (auto&& pt: points_qr) {
+                        pt /= cur_scale;
+                    }
+
+                    if (use_nn_detector_)
+                        points_qr = aligner.warpBack(points_qr);
+                    for (int j = 0; j < 4; ++j) {
+                        point.at<float>(j, 0) = points_qr[j].x;
+                        point.at<float>(j, 1) = points_qr[j].y;
+                    }
+                    // try to find duplicate qr corners
+                    bool isDuplicate = false;
+                    for (const auto &tmp_points: check_points) {
+                        const float eps = 10.f;
+                        for (size_t j = 0; j < tmp_points.size(); j++) {
+                            if (abs(tmp_points[j].x - points_qr[j].x) < eps &&
+                                abs(tmp_points[j].y - points_qr[j].y) < eps) {
+                                isDuplicate = true;
+                            }
+                            else {
+                                isDuplicate = false;
+                                break;
+                            }
+                        }
+                    }
+                    if (isDuplicate == false) {
+                        points.push_back(point);
+                        check_points.push_back(points_qr);
+                    }
+                    else {
+                        decode_results.erase(decode_results.begin() + i, decode_results.begin() + i + 1);
+                    }
                 }
-
-                if (use_nn_detector_)
-                    points_qr = aligner.warpBack(points_qr);
-                for (int i = 0; i < 4; ++i) {
-                    point.at<float>(i, 0) = points_qr[i].x;
-                    point.at<float>(i, 1) = points_qr[i].y;
-                }
-                decode_results.push_back(result);
-                points.push_back(point);
                 break;
             }
         }
diff --git a/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp b/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.cpp
@@ -34,33 +34,33 @@ QRCodeReader::QRCodeReader() : decoder_() {
     smoothMaxMultiple_ = 40;
 }
 
-Ref<Result> QRCodeReader::decode(Ref<BinaryBitmap> image) { return decode(image, DecodeHints()); }
+vector<Ref<Result>> QRCodeReader::decode(Ref<BinaryBitmap> image) { return decode(image, DecodeHints()); }
 
-Ref<Result> QRCodeReader::decode(Ref<BinaryBitmap> image, DecodeHints hints) {
+vector<Ref<Result>> QRCodeReader::decode(Ref<BinaryBitmap> image, DecodeHints hints) {
     // Binarize image using the Histogram Binarized method and be binarized
     ErrorHandler err_handler;
+    vector<Ref<Result>> result_list;
     Ref<BitMatrix> imageBitMatrix = image->getBlackMatrix(err_handler);
-    if (err_handler.ErrCode() || imageBitMatrix == NULL) return Ref<Result>();
+    if (err_handler.ErrCode() || imageBitMatrix == NULL) return result_list;
 
-    Ref<Result> rst = decodeMore(image, imageBitMatrix, hints, err_handler);
-    if (err_handler.ErrCode() || rst == NULL) {
+    vector<Ref<Result>> rst = decodeMore(image, imageBitMatrix, hints, err_handler);
+    if (err_handler.ErrCode() || rst.empty()) {
         // black white mirro!!!
         Ref<BitMatrix> invertedMatrix = image->getInvertedMatrix(err_handler);
-        if (err_handler.ErrCode() || invertedMatrix == NULL) return Ref<Result>();
-        Ref<Result> tmp_rst = decodeMore(image, invertedMatrix, hints, err_handler);
-        if (err_handler.ErrCode() || tmp_rst == NULL) return Ref<Result>();
-        return tmp_rst;
+        if (err_handler.ErrCode() || invertedMatrix == NULL) return result_list;
+        vector<Ref<Result>> tmp_rst = decodeMore(image, invertedMatrix, hints, err_handler);
+        if (err_handler.ErrCode() || tmp_rst.empty()) return tmp_rst;
     }
 
     return rst;
 }
 
-Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> imageBitMatrix,
+vector<Ref<Result>> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> imageBitMatrix,
                                      DecodeHints hints, ErrorHandler &err_handler) {
     nowHints_ = hints;
     std::string ept;
-
-    if (imageBitMatrix == NULL) return Ref<Result>();
+    vector<Ref<Result>> result_list;
+    if (imageBitMatrix == NULL) return result_list;
     image->m_poUnicomBlock->Init();
     image->m_poUnicomBlock->Reset(imageBitMatrix);
 
@@ -88,7 +88,7 @@ Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> ima
             Ref<FinderPatternInfo> patternInfo = detector->getFinderPatternInfo(i);
             setPatternFix(patternInfo->getPossibleFix());
             if (patternInfo->getAnglePossibleFix() < 0.6 && i) continue;
-
+            bool patternFoundFlag = false;
             int possibleAlignmentCount = 0;
             possibleAlignmentCount = detector->getPossibleAlignmentCount(i);
             if (possibleAlignmentCount < 0) continue;
@@ -99,6 +99,7 @@ Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> ima
 
             vector<bool> needTryVariousDeimensions(possibleAlignmentCount, false);
             for (int j = 0; j < possibleAlignmentCount; j++) {
+                if (patternFoundFlag){break;}
                 ArrayRef<Ref<ResultPoint> > points;
                 err_handler.Reset();
                 Ref<DetectorResult> detectorResult =
@@ -142,11 +143,15 @@ Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> ima
                                decoderResult->getCharset(), decoderResult->getQRCodeVersion(),
                                decoderResult->getEcLevel(), decoderResult->getCharsetMode()));
                 setSuccFix(points);
-
-                return result;
+                result_list.push_back(result);
+                patternFoundFlag = true;
+                if (nowHints_.getUseNNDetector()) {
+                    return result_list;
+                }
             }
             // try different dimentions
             for (int j = 0; j < possibleAlignmentCount; j++) {
+                if (patternFoundFlag){break;}
                 err_handler.Reset();
                 ArrayRef<Ref<ResultPoint> > points;
                 if (needTryVariousDeimensions[j]) {
@@ -188,13 +193,17 @@ Ref<Result> QRCodeReader::decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> ima
                             decoderResult->getEcLevel(), decoderResult->getCharsetMode()));
 
                         setSuccFix(points);
-                        return result;
+                        result_list.push_back(result);
+                        patternFoundFlag = true;
+                        if (nowHints_.getUseNNDetector()) {
+                            return result_list;
+                        }
                     }
                 }
             }
         }
     }
-    return Ref<Result>();
+    return result_list;
 }
 
 vector<int> QRCodeReader::getPossibleDimentions(int detectDimension) {
diff --git a/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.hpp b/modules/wechat_qrcode/src/zxing/qrcode/qrcode_reader.hpp
@@ -75,10 +75,10 @@ class QRCodeReader : public Reader {
     virtual ~QRCodeReader();
     string name() override { return "qrcode"; }
 
-    Ref<Result> decode(Ref<BinaryBitmap> image) override;
-    Ref<Result> decode(Ref<BinaryBitmap> image, DecodeHints hints) override;
+    vector<Ref<Result>> decode(Ref<BinaryBitmap> image) override;
+    vector<Ref<Result>> decode(Ref<BinaryBitmap> image, DecodeHints hints) override;
 
-    Ref<Result> decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> imageBitMatrix,
+    vector<Ref<Result>> decodeMore(Ref<BinaryBitmap> image, Ref<BitMatrix> imageBitMatrix,
                            DecodeHints hints, ErrorHandler& err_handler);
 
 private:
diff --git a/modules/wechat_qrcode/src/zxing/reader.cpp b/modules/wechat_qrcode/src/zxing/reader.cpp
diff --git a/modules/wechat_qrcode/src/zxing/reader.hpp b/modules/wechat_qrcode/src/zxing/reader.hpp
diff --git a/modules/wechat_qrcode/test/test_qrcode.cpp b/modules/wechat_qrcode/test/test_qrcode.cpp
