Skip to content

Commit ee12bc1

Browse files
author
dmitriy.anisimov
committed
added MNIST dataset loader
1 parent 9ee9f18 commit ee12bc1

File tree

4 files changed

+321
-2
lines changed

4 files changed

+321
-2
lines changed

‎modules/datasetstools/doc/datasetstools.rst‎

Lines changed: 16 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@ datasetstools. Tools for working with different datasets.
66

77
The datasetstools module includes classes for working with different datasets.
88

9-
First version of this module was implemented for **Fall2014 OpenCV Challenge**.
10-
119
Action Recognition
1210
------------------
1311

@@ -241,6 +239,22 @@ Currently implemented loading full list with urls. Planned to implement dataset
241239

242240
3. To load data run: ./opencv/build/bin/example_datasetstools_or_imagenet -p=/home/user/path_to_unpacked_file/
243241

242+
OR_mnist
243+
===========
244+
.. ocv:class:: OR_mnist
245+
246+
Implements loading dataset:
247+
248+
_`"MNIST"`: http://yann.lecun.com/exdb/mnist/
249+
250+
.. note:: Usage
251+
252+
1. From link above download dataset files: t10k-images-idx3-ubyte.gz, t10k-labels-idx1-ubyte.gz, train-images-idx3-ubyte.gz, train-labels-idx1-ubyte.gz.
253+
254+
2. Unpack them.
255+
256+
3. To load data run: ./opencv/build/bin/example_datasetstools_or_mnist -p=/home/user/path_to_unpacked_files/
257+
244258
OR_sun
245259
======
246260
.. ocv:class:: OR_sun

‎modules/datasetstools/include/opencv2/datasetstools/or_mnist.hpp‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,74 @@
1+
/*M///////////////////////////////////////////////////////////////////////////////////////
2+
//
3+
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
4+
//
5+
// By downloading, copying, installing or using the software you agree to this license.
6+
// If you do not agree to this license, do not download, install,
7+
// copy or use the software.
8+
//
9+
//
10+
// License Agreement
11+
// For Open Source Computer Vision Library
12+
//
13+
// Copyright (C) 2014, Itseez Inc, all rights reserved.
14+
// Third party copyrights are property of their respective owners.
15+
//
16+
// Redistribution and use in source and binary forms, with or without modification,
17+
// are permitted provided that the following conditions are met:
18+
//
19+
// * Redistribution's of source code must retain the above copyright notice,
20+
// this list of conditions and the following disclaimer.
21+
//
22+
// * Redistribution's in binary form must reproduce the above copyright notice,
23+
// this list of conditions and the following disclaimer in the documentation
24+
// and/or other materials provided with the distribution.
25+
//
26+
// * The name of the copyright holders may not be used to endorse or promote products
27+
// derived from this software without specific prior written permission.
28+
//
29+
// This software is provided by the copyright holders and contributors "as is" and
30+
// any express or implied warranties, including, but not limited to, the implied
31+
// warranties of merchantability and fitness for a particular purpose are disclaimed.
32+
// In no event shall the Itseez Inc or contributors be liable for any direct,
33+
// indirect, incidental, special, exemplary, or consequential damages
34+
// (including, but not limited to, procurement of substitute goods or services;
35+
// loss of use, data, or profits; or business interruption) however caused
36+
// and on any theory of liability, whether in contract, strict liability,
37+
// or tort (including negligence or otherwise) arising in any way out of
38+
// the use of this software, even if advised of the possibility of such damage.
39+
//
40+
//M*/
41+
42+
#ifndef OPENCV_DATASETSTOOLS_OR_MNIST_HPP
43+
#define OPENCV_DATASETSTOOLS_OR_MNIST_HPP
44+
45+
#include <string>
46+
#include <vector>
47+
48+
#include "opencv2/datasetstools/dataset.hpp"
49+
50+
#include <opencv2/core.hpp>
51+
52+
namespace cv
53+
{
54+
namespace datasetstools
55+
{
56+
57+
struct OR_mnistObj : public Object
58+
{
59+
char label; // 0..9
60+
Mat image; // [28][28]
61+
};
62+
63+
class CV_EXPORTS OR_mnist : public Dataset
64+
{
65+
public:
66+
virtual void load(const std::string &path) = 0;
67+
68+
static Ptr<OR_mnist> create();
69+
};
70+
71+
}
72+
}
73+
74+
#endif

‎modules/datasetstools/samples/or_mnist.cpp‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
/*M///////////////////////////////////////////////////////////////////////////////////////
2+
//
3+
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
4+
//
5+
// By downloading, copying, installing or using the software you agree to this license.
6+
// If you do not agree to this license, do not download, install,
7+
// copy or use the software.
8+
//
9+
//
10+
// License Agreement
11+
// For Open Source Computer Vision Library
12+
//
13+
// Copyright (C) 2014, Itseez Inc, all rights reserved.
14+
// Third party copyrights are property of their respective owners.
15+
//
16+
// Redistribution and use in source and binary forms, with or without modification,
17+
// are permitted provided that the following conditions are met:
18+
//
19+
// * Redistribution's of source code must retain the above copyright notice,
20+
// this list of conditions and the following disclaimer.
21+
//
22+
// * Redistribution's in binary form must reproduce the above copyright notice,
23+
// this list of conditions and the following disclaimer in the documentation
24+
// and/or other materials provided with the distribution.
25+
//
26+
// * The name of the copyright holders may not be used to endorse or promote products
27+
// derived from this software without specific prior written permission.
28+
//
29+
// This software is provided by the copyright holders and contributors "as is" and
30+
// any express or implied warranties, including, but not limited to, the implied
31+
// warranties of merchantability and fitness for a particular purpose are disclaimed.
32+
// In no event shall the Itseez Inc or contributors be liable for any direct,
33+
// indirect, incidental, special, exemplary, or consequential damages
34+
// (including, but not limited to, procurement of substitute goods or services;
35+
// loss of use, data, or profits; or business interruption) however caused
36+
// and on any theory of liability, whether in contract, strict liability,
37+
// or tort (including negligence or otherwise) arising in any way out of
38+
// the use of this software, even if advised of the possibility of such damage.
39+
//
40+
//M*/
41+
42+
#include "opencv2/datasetstools/or_mnist.hpp"
43+
44+
#include <opencv2/core.hpp>
45+
#include <opencv2/imgcodecs.hpp>
46+
47+
#include <cstdio>
48+
49+
#include <string>
50+
#include <vector>
51+
52+
using namespace std;
53+
using namespace cv;
54+
using namespace cv::datasetstools;
55+
56+
int main(int argc, char *argv[])
57+
{
58+
const char *keys =
59+
"{ help h usage ? | | show this message }"
60+
"{ path p |true| path to dataset (SUN397 folder) }";
61+
CommandLineParser parser(argc, argv, keys);
62+
string path(parser.get<string>("path"));
63+
if (parser.has("help") || path=="true")
64+
{
65+
parser.printMessage();
66+
return -1;
67+
}
68+
69+
Ptr<OR_mnist> dataset = OR_mnist::create();
70+
dataset->load(path);
71+
72+
// ***************
73+
// dataset contains for each object its image and label.
74+
// For example, let output train & test sizes and their first elements.
75+
printf("train size: %u\n", (unsigned int)dataset->getTrain().size());
76+
printf("test size: %u\n", (unsigned int)dataset->getTest().size());
77+
78+
OR_mnistObj *example = static_cast<OR_mnistObj *>(dataset->getTrain()[0].get());
79+
printf("first train:\nlabel: %u\n", example->label);
80+
printf("image was saved to train0.png\n");
81+
imwrite("train0.png", example->image);
82+
83+
example = static_cast<OR_mnistObj *>(dataset->getTest()[0].get());
84+
printf("first test:\nlabel: %u\n", example->label);
85+
printf("image was saved to test0.png\n");
86+
imwrite("test0.png", example->image);
87+
88+
return 0;
89+
}

‎modules/datasetstools/src/or_mnist.cpp‎

Lines changed: 142 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
/*M///////////////////////////////////////////////////////////////////////////////////////
2+
//
3+
// IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING.
4+
//
5+
// By downloading, copying, installing or using the software you agree to this license.
6+
// If you do not agree to this license, do not download, install,
7+
// copy or use the software.
8+
//
9+
//
10+
// License Agreement
11+
// For Open Source Computer Vision Library
12+
//
13+
// Copyright (C) 2014, Itseez Inc, all rights reserved.
14+
// Third party copyrights are property of their respective owners.
15+
//
16+
// Redistribution and use in source and binary forms, with or without modification,
17+
// are permitted provided that the following conditions are met:
18+
//
19+
// * Redistribution's of source code must retain the above copyright notice,
20+
// this list of conditions and the following disclaimer.
21+
//
22+
// * Redistribution's in binary form must reproduce the above copyright notice,
23+
// this list of conditions and the following disclaimer in the documentation
24+
// and/or other materials provided with the distribution.
25+
//
26+
// * The name of the copyright holders may not be used to endorse or promote products
27+
// derived from this software without specific prior written permission.
28+
//
29+
// This software is provided by the copyright holders and contributors "as is" and
30+
// any express or implied warranties, including, but not limited to, the implied
31+
// warranties of merchantability and fitness for a particular purpose are disclaimed.
32+
// In no event shall the Itseez Inc or contributors be liable for any direct,
33+
// indirect, incidental, special, exemplary, or consequential damages
34+
// (including, but not limited to, procurement of substitute goods or services;
35+
// loss of use, data, or profits; or business interruption) however caused
36+
// and on any theory of liability, whether in contract, strict liability,
37+
// or tort (including negligence or otherwise) arising in any way out of
38+
// the use of this software, even if advised of the possibility of such damage.
39+
//
40+
//M*/
41+
42+
#include "opencv2/datasetstools/or_mnist.hpp"
43+
#include "precomp.hpp"
44+
45+
namespace cv
46+
{
47+
namespace datasetstools
48+
{
49+
50+
using namespace std;
51+
52+
class CV_EXPORTS OR_mnistImp : public OR_mnist
53+
{
54+
public:
55+
OR_mnistImp() {}
56+
//OR_mnistImp(const string &path);
57+
virtual ~OR_mnistImp() {}
58+
59+
virtual void load(const string &path);
60+
61+
private:
62+
void loadDataset(const string &path);
63+
64+
void loadDatasetPart(const string &imagesFile, const string &labelsFile, unsigned int num, vector< Ptr<Object> > &dataset_);
65+
};
66+
67+
/*OR_mnistImp::OR_mnistImp(const string &path)
68+
{
69+
loadDataset(path);
70+
}*/
71+
72+
void OR_mnistImp::load(const string &path)
73+
{
74+
loadDataset(path);
75+
}
76+
77+
void OR_mnistImp::loadDatasetPart(const string &imagesFile, const string &labelsFile, unsigned int num, vector< Ptr<Object> > &dataset_)
78+
{
79+
FILE *f = fopen(imagesFile.c_str(), "rb");
80+
fseek(f, 16, SEEK_CUR);
81+
unsigned int imageSize = 28*28;
82+
char *images = new char[num*imageSize];
83+
size_t res = fread(images, 1, num*imageSize, f);
84+
fclose(f);
85+
if (num*imageSize != res)
86+
{
87+
return;
88+
}
89+
f = fopen(labelsFile.c_str(), "rb");
90+
fseek(f, 8, SEEK_CUR);
91+
char *labels = new char[num];
92+
res = fread(labels, 1, num, f);
93+
fclose(f);
94+
if (num != res)
95+
{
96+
return;
97+
}
98+
99+
for (unsigned int i=0; i<num; ++i)
100+
{
101+
Ptr<OR_mnistObj> curr(new OR_mnistObj);
102+
curr->label = labels[i];
103+
104+
curr->image = Mat(28, 28, CV_8U);
105+
unsigned int imageIdx = i*imageSize;
106+
for (int j=0; j<curr->image.rows; ++j)
107+
{
108+
char *im = curr->image.ptr<char>(j);
109+
for (int k=0; k<curr->image.cols; ++k)
110+
{
111+
im[k] = images[imageIdx + j*28 + k];
112+
}
113+
}
114+
115+
dataset_.push_back(curr);
116+
}
117+
delete[] images;
118+
delete[] labels;
119+
}
120+
121+
void OR_mnistImp::loadDataset(const string &path)
122+
{
123+
train.push_back(vector< Ptr<Object> >());
124+
test.push_back(vector< Ptr<Object> >());
125+
validation.push_back(vector< Ptr<Object> >());
126+
127+
string trainImagesFile(path + "train-images.idx3-ubyte");
128+
string trainLabelsFile(path + "train-labels.idx1-ubyte");
129+
loadDatasetPart(trainImagesFile, trainLabelsFile, 60000, train.back());
130+
131+
string testImagesFile(path + "t10k-images.idx3-ubyte");
132+
string testLabelsFile(path + "t10k-labels.idx1-ubyte");
133+
loadDatasetPart(testImagesFile, testLabelsFile, 10000, test.back());
134+
}
135+
136+
Ptr<OR_mnist> OR_mnist::create()
137+
{
138+
return Ptr<OR_mnistImp>(new OR_mnistImp);
139+
}
140+
141+
}
142+
}

0 commit comments

Comments
 (0)