Skip to content

Commit 25227f1

Browse files
fran6cofran6co
committed
New VideoToolbox rgb packet processor
Mac OS X >= 10.8 has hardware accelerated jpeg decoding (a bit hidden)
1 parent 01a12f8 commit 25227f1

File tree

5 files changed

+261
-10
lines changed

5 files changed

+261
-10
lines changed

‎examples/protonect/CMakeLists.txt

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,6 +169,29 @@ IF(ENABLE_OPENCL)
169169
ENDIF(OPENCL_FOUND)
170170
ENDIF(ENABLE_OPENCL)
171171

172+
IF(APPLE)
173+
FIND_LIBRARY(VIDEOTOOLBOX_LIBRARY VideoToolbox)
174+
175+
IF(VIDEOTOOLBOX_LIBRARY)
176+
SET(LIBFREENECT2_WITH_VT_SUPPORT 1)
177+
178+
FIND_LIBRARY(COREFOUNDATION_LIBRARY CoreFoundation REQUIRED)
179+
FIND_LIBRARY(COREMEDIA_LIBRARY CoreMedia REQUIRED)
180+
FIND_LIBRARY(COREVIDEO_LIBRARY CoreVideo REQUIRED)
181+
182+
LIST(APPEND SOURCES
183+
src/vt_rgb_packet_processor.cpp
184+
)
185+
186+
LIST(APPEND LIBRARIES
187+
${VIDEOTOOLBOX_LIBRARY}
188+
${COREFOUNDATION_LIBRARY}
189+
${COREMEDIA_LIBRARY}
190+
${COREVIDEO_LIBRARY}
191+
)
192+
ENDIF(VIDEOTOOLBOX_LIBRARY)
193+
ENDIF(APPLE)
194+
172195
SET(CMAKE_INSTALL_RPATH ${LibUSB_LIBDIR})
173196

174197
CONFIGURE_FILE("${MY_DIR}/include/libfreenect2/config.h.in" "${MY_DIR}/include/libfreenect2/config.h" @ONLY)

‎examples/protonect/include/libfreenect2/config.h.in

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@
4343

4444
#cmakedefine LIBFREENECT2_WITH_OPENCL_SUPPORT
4545

46+
#cmakedefine LIBFREENECT2_WITH_VT_SUPPORT
47+
4648
#cmakedefine LIBFREENECT2_THREADING_STDLIB
4749

4850
#cmakedefine LIBFREENECT2_THREADING_TINYTHREAD

‎examples/protonect/include/libfreenect2/frame_listener.hpp

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -49,22 +49,30 @@ class LIBFREENECT2_API Frame
4949
size_t width, height, bytes_per_pixel;
5050
unsigned char* data;
5151

52-
Frame(size_t width, size_t height, size_t bytes_per_pixel) :
52+
Frame(size_t width, size_t height, size_t bytes_per_pixel, unsigned char* _data = NULL) :
5353
width(width),
5454
height(height),
55-
bytes_per_pixel(bytes_per_pixel)
55+
bytes_per_pixel(bytes_per_pixel),
56+
rawdata(NULL),
57+
data(_data)
5658
{
57-
const size_t alignment = 64;
58-
size_t space = width * height * bytes_per_pixel + alignment;
59-
rawdata = new unsigned char[space];
60-
uintptr_t ptr = reinterpret_cast<uintptr_t>(rawdata);
61-
uintptr_t aligned = (ptr - 1u + alignment) & -alignment;
62-
data = reinterpret_cast<unsigned char *>(aligned);
59+
if (data == NULL)
60+
{
61+
const size_t alignment = 64;
62+
size_t space = width * height * bytes_per_pixel + alignment;
63+
rawdata = new unsigned char[space];
64+
uintptr_t ptr = reinterpret_cast<uintptr_t>(rawdata);
65+
uintptr_t aligned = (ptr - 1u + alignment) & -alignment;
66+
data = reinterpret_cast<unsigned char *>(aligned);
67+
}
6368
}
6469

65-
~Frame()
70+
virtual ~Frame()
6671
{
67-
delete[] rawdata;
72+
if (rawdata)
73+
{
74+
delete[] rawdata;
75+
}
6876
}
6977

7078
protected:

‎examples/protonect/include/libfreenect2/rgb_packet_processor.h

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -81,5 +81,20 @@ class LIBFREENECT2_API TurboJpegRgbPacketProcessor : public RgbPacketProcessor
8181
TurboJpegRgbPacketProcessorImpl *impl_;
8282
};
8383

84+
#ifdef LIBFREENECT2_WITH_VT_SUPPORT
85+
class VTRgbPacketProcessorImpl;
86+
87+
class LIBFREENECT2_API VTRgbPacketProcessor : public RgbPacketProcessor
88+
{
89+
public:
90+
VTRgbPacketProcessor();
91+
virtual ~VTRgbPacketProcessor();
92+
protected:
93+
virtual void process(const libfreenect2::RgbPacket &packet);
94+
private:
95+
VTRgbPacketProcessorImpl *impl_;
96+
};
97+
#endif
98+
8499
} /* namespace libfreenect2 */
85100
#endif /* RGB_PACKET_PROCESSOR_H_ */

‎examples/protonect/src/vt_rgb_packet_processor.cpp

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
/*
2+
* This file is part of the OpenKinect Project. http://www.openkinect.org
3+
*
4+
* Copyright (c) 2014 individual OpenKinect contributors. See the CONTRIB file
5+
* for details.
6+
*
7+
* This code is licensed to you under the terms of the Apache License, version
8+
* 2.0, or, at your option, the terms of the GNU General Public License,
9+
* version 2.0. See the APACHE20 and GPL2 files for the text of the licenses,
10+
* or the following URLs:
11+
* http://www.apache.org/licenses/LICENSE-2.0
12+
* http://www.gnu.org/licenses/gpl-2.0.txt
13+
*
14+
* If you redistribute this file in source form, modified or unmodified, you
15+
* may:
16+
* 1) Leave this header intact and distribute it under the same terms,
17+
* accompanying it with the APACHE20 and GPL20 files, or
18+
* 2) Delete the Apache 2.0 clause and accompany it with the GPL2 file, or
19+
* 3) Delete the GPL v2 clause and accompany it with the APACHE20 file
20+
* In all cases you must keep the copyright notice intact and include a copy
21+
* of the CONTRIB file.
22+
*
23+
* Binary distributions must follow the binary distribution requirements of
24+
* either License.
25+
*/
26+
27+
28+
#include <libfreenect2/rgb_packet_processor.h>
29+
30+
#include <VideoToolbox/VideoToolbox.h>
31+
#include <iostream>
32+
33+
namespace libfreenect2 {
34+
35+
class VTFrame: public Frame
36+
{
37+
public:
38+
VTFrame(size_t width, size_t height, size_t bytes_per_pixel, CVPixelBufferRef pixelBuffer) :
39+
Frame(width,
40+
height,
41+
bytes_per_pixel,
42+
reinterpret_cast<unsigned char *>(CVPixelBufferGetBaseAddress(lockPixelBuffer(pixelBuffer)))),
43+
pixelBuffer(pixelBuffer) {
44+
}
45+
46+
~VTFrame() {
47+
CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
48+
CVPixelBufferRelease(pixelBuffer);
49+
}
50+
51+
protected:
52+
CVPixelBufferRef lockPixelBuffer(CVPixelBufferRef _pixelBuffer) {
53+
CVPixelBufferLockBaseAddress(_pixelBuffer, 0);
54+
55+
return _pixelBuffer;
56+
}
57+
58+
CVPixelBufferRef pixelBuffer;
59+
};
60+
61+
class VTRgbPacketProcessorImpl
62+
{
63+
public:
64+
CMFormatDescriptionRef format;
65+
VTDecompressionSessionRef decoder;
66+
67+
double timing_acc;
68+
double timing_acc_n;
69+
70+
Timer timer;
71+
72+
VTRgbPacketProcessorImpl() {
73+
timing_acc = 0.0;
74+
timing_acc_n = 0.0;
75+
76+
int32_t width = 1920, height = 1080;
77+
78+
CMVideoFormatDescriptionCreate(NULL, kCMVideoCodecType_JPEG, width, height, nil, &format);
79+
80+
const void *decoderSpecificationKeys[] = {kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder};
81+
const void *decoderSpecificationValues[] = {kCFBooleanTrue};
82+
CFDictionaryRef decoderSpecification = CFDictionaryCreate(NULL,
83+
decoderSpecificationKeys,
84+
decoderSpecificationValues,
85+
1,
86+
&kCFTypeDictionaryKeyCallBacks,
87+
&kCFTypeDictionaryValueCallBacks);
88+
89+
int32_t pixelFormat = kCVPixelFormatType_32BGRA;
90+
const void *outputKeys[] = {kCVPixelBufferPixelFormatTypeKey, kCVPixelBufferWidthKey, kCVPixelBufferHeightKey};
91+
const void *outputValues[] =
92+
{CFNumberCreate(NULL, kCFNumberSInt32Type, &pixelFormat), CFNumberCreate(NULL, kCFNumberSInt32Type, &width),
93+
CFNumberCreate(NULL, kCFNumberSInt32Type, &height)};
94+
95+
CFDictionaryRef outputConfiguration = CFDictionaryCreate(NULL,
96+
outputKeys,
97+
outputValues,
98+
3,
99+
&kCFTypeDictionaryKeyCallBacks,
100+
&kCFTypeDictionaryValueCallBacks);
101+
102+
VTDecompressionOutputCallbackRecord callback = {&VTRgbPacketProcessorImpl::decodeFrame, NULL};
103+
104+
VTDecompressionSessionCreate(NULL, format, decoderSpecification, outputConfiguration, &callback, &decoder);
105+
106+
CFRelease(decoderSpecification);
107+
CFRelease(outputConfiguration);
108+
}
109+
110+
~VTRgbPacketProcessorImpl() {
111+
VTDecompressionSessionInvalidate(decoder);
112+
CFRelease(decoder);
113+
CFRelease(format);
114+
}
115+
116+
static void decodeFrame(void *decompressionOutputRefCon,
117+
void *sourceFrameRefCon,
118+
OSStatus status,
119+
VTDecodeInfoFlags infoFlags,
120+
CVImageBufferRef pixelBuffer,
121+
CMTime presentationTimeStamp,
122+
CMTime presentationDuration) {
123+
CVPixelBufferRef *outputPixelBuffer = (CVPixelBufferRef *) sourceFrameRefCon;
124+
*outputPixelBuffer = CVPixelBufferRetain(pixelBuffer);
125+
}
126+
127+
void startTiming() {
128+
timer.start();
129+
}
130+
131+
void stopTiming() {
132+
timing_acc += timer.stop();
133+
timing_acc_n += 1.0;
134+
135+
if (timing_acc_n >= 100.0) {
136+
double avg = (timing_acc / timing_acc_n);
137+
std::cout << "[VTRgbPacketProcessor] avg. time: " << (avg * 1000) << "ms -> ~" << (1.0 / avg) << "Hz"
138+
<< std::endl;
139+
timing_acc = 0.0;
140+
timing_acc_n = 0.0;
141+
}
142+
}
143+
};
144+
145+
VTRgbPacketProcessor::VTRgbPacketProcessor() :
146+
impl_(new VTRgbPacketProcessorImpl())
147+
{
148+
}
149+
150+
VTRgbPacketProcessor::~VTRgbPacketProcessor()
151+
{
152+
delete impl_;
153+
}
154+
155+
void VTRgbPacketProcessor::process(const RgbPacket &packet)
156+
{
157+
if (listener_ != 0) {
158+
impl_->startTiming();
159+
160+
CMBlockBufferRef blockBuffer;
161+
CMBlockBufferCreateWithMemoryBlock(
162+
NULL,
163+
packet.jpeg_buffer,
164+
packet.jpeg_buffer_length,
165+
kCFAllocatorNull,
166+
NULL,
167+
0,
168+
packet.jpeg_buffer_length,
169+
0,
170+
&blockBuffer
171+
);
172+
173+
CMSampleBufferRef sampleBuffer;
174+
CMSampleBufferCreateReady(
175+
NULL,
176+
blockBuffer,
177+
impl_->format,
178+
1,
179+
0,
180+
NULL,
181+
0,
182+
NULL,
183+
&sampleBuffer
184+
);
185+
186+
CVPixelBufferRef pixelBuffer = NULL;
187+
VTDecompressionSessionDecodeFrame(impl_->decoder, sampleBuffer, 0, &pixelBuffer, NULL);
188+
189+
Frame *frame = new VTFrame(1920, 1080, 4, pixelBuffer);
190+
191+
frame->timestamp = packet.timestamp;
192+
frame->sequence = packet.sequence;
193+
194+
listener_->onNewFrame(Frame::Color, frame);
195+
196+
CFRelease(sampleBuffer);
197+
CFRelease(blockBuffer);
198+
199+
impl_->stopTiming();
200+
}
201+
}
202+
203+
} /* namespace libfreenect2 */

0 commit comments

Comments
 (0)