New VideoToolbox rgb packet processor

Mac OS X >= 10.8 has hardware accelerated jpeg decoding (a bit hidden)

Author: fran6co

CMakeLists.txt

@@ -76,7 +76,6 @@ SET(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 # dependencies
 FIND_PACKAGE(PkgConfig)    # try find PKGConfig as it will be used if found
 FIND_PACKAGE(LibUSB REQUIRED)
-FIND_PACKAGE(TurboJPEG REQUIRED) #does not provide a package-config file
 
 # Add includes
 INCLUDE_DIRECTORIES(
@@ -85,7 +84,6 @@ INCLUDE_DIRECTORIES(
   ${PROJECT_BINARY_DIR} # for generated headers
   ${LIBFREENECT2_THREADING_INCLUDE_DIR}
   ${LibUSB_INCLUDE_DIRS}
-  ${TurboJPEG_INCLUDE_DIRS}
 )
 
 SET(RESOURCES_INC_FILE "${PROJECT_BINARY_DIR}/resources.inc.h")
@@ -125,7 +123,6 @@ SET(SOURCES
   src/packet_pipeline.cpp
   src/rgb_packet_stream_parser.cpp
   src/rgb_packet_processor.cpp
-  src/turbo_jpeg_rgb_packet_processor.cpp
   src/depth_packet_stream_parser.cpp
   src/depth_packet_processor.cpp
   src/cpu_depth_packet_processor.cpp
@@ -143,15 +140,61 @@ SET(SOURCES
 
 SET(LIBRARIES
   ${LibUSB_LIBRARIES}
-  ${TurboJPEG_LIBRARIES}
   ${LIBFREENECT2_THREADING_LIBRARIES}
 )
 
 SET(LIBFREENECT2_DLLS
   ${LibUSB_DLL}
-  ${TurboJPEG_DLL}
 )
 
+IF(APPLE)
+  FIND_LIBRARY(VIDEOTOOLBOX_LIBRARY VideoToolbox)
+
+  IF(VIDEOTOOLBOX_LIBRARY)
+    SET(LIBFREENECT2_WITH_VT_SUPPORT 1)
+
+    FIND_LIBRARY(COREFOUNDATION_LIBRARY CoreFoundation REQUIRED)
+    FIND_LIBRARY(COREMEDIA_LIBRARY CoreMedia REQUIRED)
+    FIND_LIBRARY(COREVIDEO_LIBRARY CoreVideo REQUIRED)
+
+    LIST(APPEND SOURCES
+      src/vt_rgb_packet_processor.cpp
+    )
+
+    LIST(APPEND LIBRARIES
+      ${VIDEOTOOLBOX_LIBRARY}
+      ${COREFOUNDATION_LIBRARY}
+      ${COREMEDIA_LIBRARY}
+      ${COREVIDEO_LIBRARY}
+    )
+  ENDIF(VIDEOTOOLBOX_LIBRARY)
+ENDIF(APPLE)
+
+IF(LIBFREENECT2_WITH_VT_SUPPORT)
+  FIND_PACKAGE(TurboJPEG)
+ELSE()
+  FIND_PACKAGE(TurboJPEG REQUIRED)
+ENDIF()
+
+IF(TurboJPEG_FOUND)
+  SET(LIBFREENECT2_WITH_TURBOJPEG_SUPPORT 1)
+
+  INCLUDE_DIRECTORIES(${TurboJPEG_INCLUDE_DIRS})
+
+  LIST(APPEND SOURCES
+    src/turbo_jpeg_rgb_packet_processor.cpp
+  )
+
+  LIST(APPEND LIBRARIES
+    ${TurboJPEG_LIBRARIES}
+  )
+
+  LIST(APPEND LIBFREENECT2_DLLS
+   ${TurboJPEG_DLL}
+  )
+
+ENDIF()
+
 IF(ENABLE_OPENGL)
   FIND_PACKAGE(GLFW3)
   FIND_PACKAGE(OpenGL)

README.md

@@ -189,13 +189,12 @@ Use your favorite package managers (brew, ports, etc.) to install most if not al
 
 1. ``cd`` into a directory where you want to keep libfreenect2 stuff in
 1. Make sure these build tools are available: wget, git, cmake, pkg-config. Xcode may provide some of them. Install the rest via package managers.
-1. Install dependencies: libusb, TurboJPEG, GLFW, OpenNI2 (optional).
+1. Install dependencies: libusb, TurboJPEG (optional), GLFW, OpenNI2 (optional).
 
     ```
 brew update
 brew install libusb
 brew tap homebrew/science
-brew install jpeg-turbo
 brew tap homebrew/versions
 brew install glfw3
 brew install openni2
@@ -205,6 +204,8 @@ export OPENNI2_INCLUDE=/usr/local/include/ni2
 
     It **is** now recommended to install libusb from package managers instead of building from source locally. Previously it was not recommended but that is no longer the case.
 
+    You are still able to build with optional TurboJPEG support with ```brew install jpeg-turbo``` along with other dependencies
+
     It is not recommended to build TurboJPEG from source, which may produce corrupted results on Mac OSX according to user reports. Install TurboJPEG binary only from package managers.
 
 1. Download the libfreenect2 repository

include/internal/libfreenect2/rgb_packet_processor.h

@@ -77,6 +77,7 @@ class DumpRgbPacketProcessor : public RgbPacketProcessor
   virtual void process(const libfreenect2::RgbPacket &packet);
 };
 
+#ifdef LIBFREENECT2_WITH_TURBOJPEG_SUPPORT
 class TurboJpegRgbPacketProcessorImpl;
 
 /** Processor to decode JPEG to image, using TurboJpeg. */
@@ -90,6 +91,22 @@ class TurboJpegRgbPacketProcessor : public RgbPacketProcessor
 private:
   TurboJpegRgbPacketProcessorImpl *impl_; ///< Decoder implementation.
 };
+#endif
+
+#ifdef LIBFREENECT2_WITH_VT_SUPPORT
+class VTRgbPacketProcessorImpl;
+
+class LIBFREENECT2_API VTRgbPacketProcessor : public RgbPacketProcessor
+{
+public:
+  VTRgbPacketProcessor();
+  virtual ~VTRgbPacketProcessor();
+protected:
+  virtual void process(const libfreenect2::RgbPacket &packet);
+private:
+  VTRgbPacketProcessorImpl *impl_;
+};
+#endif
 
 } /* namespace libfreenect2 */
 #endif /* RGB_PACKET_PROCESSOR_H_ */

include/libfreenect2/config.h.in

@@ -44,6 +44,10 @@
 #cmakedefine LIBFREENECT2_WITH_OPENCL_SUPPORT
 #cmakedefine LIBFREENECT2_OPENCL_ICD_LOADER_IS_OLD
 
+#cmakedefine LIBFREENECT2_WITH_VT_SUPPORT
+
+#cmakedefine LIBFREENECT2_WITH_TURBOJPEG_SUPPORT
+
 #cmakedefine LIBFREENECT2_THREADING_STDLIB
 
 #cmakedefine LIBFREENECT2_THREADING_TINYTHREAD

src/packet_pipeline.cpp

@@ -36,6 +36,17 @@
 namespace libfreenect2
 {
 
+static RgbPacketProcessor *getDefaultRgbPacketProcessor()
+{
+  #ifdef LIBFREENECT2_WITH_VT_SUPPORT
+    return new VTRgbPacketProcessor();
+  #endif
+  #ifdef LIBFREENECT2_WITH_TURBOJPEG_SUPPORT
+    return new TurboJpegRgbPacketProcessor();
+  #endif
+  return NULL;
+}
+
 class PacketPipelineComponents
 {
 public:
@@ -104,16 +115,16 @@ DepthPacketProcessor *PacketPipeline::getDepthPacketProcessor() const
 }
 
 CpuPacketPipeline::CpuPacketPipeline()
-{ 
-  comp_->initialize(new TurboJpegRgbPacketProcessor(), new CpuDepthPacketProcessor());
+{
+  comp_->initialize(getDefaultRgbPacketProcessor(), new CpuDepthPacketProcessor());
 }
 
 CpuPacketPipeline::~CpuPacketPipeline() { }
 
 #ifdef LIBFREENECT2_WITH_OPENGL_SUPPORT
 OpenGLPacketPipeline::OpenGLPacketPipeline(void *parent_opengl_context, bool debug) : parent_opengl_context_(parent_opengl_context), debug_(debug)
-{ 
-  comp_->initialize(new TurboJpegRgbPacketProcessor(), new OpenGLDepthPacketProcessor(parent_opengl_context_, debug_));
+{
+  comp_->initialize(getDefaultRgbPacketProcessor(), new OpenGLDepthPacketProcessor(parent_opengl_context_, debug_));
 }
 
 OpenGLPacketPipeline::~OpenGLPacketPipeline() { }
@@ -123,8 +134,8 @@ OpenGLPacketPipeline::~OpenGLPacketPipeline() { }
 #ifdef LIBFREENECT2_WITH_OPENCL_SUPPORT
 
 OpenCLPacketPipeline::OpenCLPacketPipeline(const int deviceId) : deviceId(deviceId)
-{ 
-  comp_->initialize(new TurboJpegRgbPacketProcessor(), new OpenCLDepthPacketProcessor(deviceId));
+{
+  comp_->initialize(getDefaultRgbPacketProcessor(), new OpenCLDepthPacketProcessor(deviceId));
 }
 
 OpenCLPacketPipeline::~OpenCLPacketPipeline() { }

src/vt_rgb_packet_processor.cpp

@@ -0,0 +1,179 @@
+/*
+ * This file is part of the OpenKinect Project. http://www.openkinect.org
+ *
+ * Copyright (c) 2014 individual OpenKinect contributors. See the CONTRIB file
+ * for details.
+ *
+ * This code is licensed to you under the terms of the Apache License, version
+ * 2.0, or, at your option, the terms of the GNU General Public License,
+ * version 2.0. See the APACHE20 and GPL2 files for the text of the licenses,
+ * or the following URLs:
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.gnu.org/licenses/gpl-2.0.txt
+ *
+ * If you redistribute this file in source form, modified or unmodified, you
+ * may:
+ *   1) Leave this header intact and distribute it under the same terms,
+ *      accompanying it with the APACHE20 and GPL20 files, or
+ *   2) Delete the Apache 2.0 clause and accompany it with the GPL2 file, or
+ *   3) Delete the GPL v2 clause and accompany it with the APACHE20 file
+ * In all cases you must keep the copyright notice intact and include a copy
+ * of the CONTRIB file.
+ *
+ * Binary distributions must follow the binary distribution requirements of
+ * either License.
+ */
+
+
+#include <libfreenect2/rgb_packet_processor.h>
+#include <libfreenect2/logging.h>
+
+#include <VideoToolbox/VideoToolbox.h>
+
+namespace libfreenect2 {
+
+class VTFrame: public Frame
+{
+ public:
+  VTFrame(size_t width, size_t height, size_t bytes_per_pixel, CVPixelBufferRef pixelBuffer) :
+      Frame(width,
+            height,
+            bytes_per_pixel,
+            reinterpret_cast<unsigned char *>(CVPixelBufferGetBaseAddress(lockPixelBuffer(pixelBuffer)))),
+      pixelBuffer(pixelBuffer) {
+  }
+
+  ~VTFrame() {
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
+    CVPixelBufferRelease(pixelBuffer);
+  }
+
+ protected:
+  CVPixelBufferRef lockPixelBuffer(CVPixelBufferRef _pixelBuffer) {
+    CVPixelBufferLockBaseAddress(_pixelBuffer, 0);
+
+    return _pixelBuffer;
+  }
+
+  CVPixelBufferRef pixelBuffer;
+};
+
+class VTRgbPacketProcessorImpl: public WithPerfLogging
+{
+ public:
+  CMFormatDescriptionRef format;
+  VTDecompressionSessionRef decoder;
+
+  VTRgbPacketProcessorImpl() {
+    int32_t width = 1920, height = 1080;
+
+    CMVideoFormatDescriptionCreate(NULL, kCMVideoCodecType_JPEG, width, height, nil, &format);
+
+    const void *decoderSpecificationKeys[] = {kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder};
+    const void *decoderSpecificationValues[] = {kCFBooleanTrue};
+    CFDictionaryRef decoderSpecification = CFDictionaryCreate(NULL,
+                                                              decoderSpecificationKeys,
+                                                              decoderSpecificationValues,
+                                                              1,
+                                                              &kCFTypeDictionaryKeyCallBacks,
+                                                              &kCFTypeDictionaryValueCallBacks);
+
+    int32_t pixelFormat = kCVPixelFormatType_32BGRA;
+    const void *outputKeys[] = {kCVPixelBufferPixelFormatTypeKey, kCVPixelBufferWidthKey, kCVPixelBufferHeightKey};
+    const void *outputValues[] =
+        {CFNumberCreate(NULL, kCFNumberSInt32Type, &pixelFormat), CFNumberCreate(NULL, kCFNumberSInt32Type, &width),
+         CFNumberCreate(NULL, kCFNumberSInt32Type, &height)};
+
+    CFDictionaryRef outputConfiguration = CFDictionaryCreate(NULL,
+                                                             outputKeys,
+                                                             outputValues,
+                                                             3,
+                                                             &kCFTypeDictionaryKeyCallBacks,
+                                                             &kCFTypeDictionaryValueCallBacks);
+
+    VTDecompressionOutputCallbackRecord callback = {&VTRgbPacketProcessorImpl::decodeFrame, NULL};
+
+    VTDecompressionSessionCreate(NULL, format, decoderSpecification, outputConfiguration, &callback, &decoder);
+
+    CFRelease(decoderSpecification);
+    CFRelease(outputConfiguration);
+  }
+
+  ~VTRgbPacketProcessorImpl() {
+    VTDecompressionSessionInvalidate(decoder);
+    CFRelease(decoder);
+    CFRelease(format);
+  }
+
+  static void decodeFrame(void *decompressionOutputRefCon,
+                          void *sourceFrameRefCon,
+                          OSStatus status,
+                          VTDecodeInfoFlags infoFlags,
+                          CVImageBufferRef pixelBuffer,
+                          CMTime presentationTimeStamp,
+                          CMTime presentationDuration) {
+    CVPixelBufferRef *outputPixelBuffer = (CVPixelBufferRef *) sourceFrameRefCon;
+    *outputPixelBuffer = CVPixelBufferRetain(pixelBuffer);
+  }
+};
+
+VTRgbPacketProcessor::VTRgbPacketProcessor()
+    : RgbPacketProcessor ()
+    , impl_(new VTRgbPacketProcessorImpl())
+{
+}
+
+VTRgbPacketProcessor::~VTRgbPacketProcessor()
+{
+  delete impl_;
+}
+
+void VTRgbPacketProcessor::process(const RgbPacket &packet)
+{
+  if (listener_ != 0) {
+    impl_->startTiming();
+
+    CMBlockBufferRef blockBuffer;
+    CMBlockBufferCreateWithMemoryBlock(
+        NULL,
+        packet.jpeg_buffer,
+        packet.jpeg_buffer_length,
+        kCFAllocatorNull,
+        NULL,
+        0,
+        packet.jpeg_buffer_length,
+        0,
+        &blockBuffer
+    );
+
+    CMSampleBufferRef sampleBuffer;
+    CMSampleBufferCreateReady(
+        NULL,
+        blockBuffer,
+        impl_->format,
+        1,
+        0,
+        NULL,
+        0,
+        NULL,
+        &sampleBuffer
+    );
+
+    CVPixelBufferRef pixelBuffer = NULL;
+    VTDecompressionSessionDecodeFrame(impl_->decoder, sampleBuffer, 0, &pixelBuffer, NULL);
+
+    Frame *frame = new VTFrame(1920, 1080, 4, pixelBuffer);
+
+    frame->timestamp = packet.timestamp;
+    frame->sequence = packet.sequence;
+
+    listener_->onNewFrame(Frame::Color, frame);
+
+    CFRelease(sampleBuffer);
+    CFRelease(blockBuffer);
+
+    impl_->stopTiming(LOG_INFO);
+  }
+}
+
+} /* namespace libfreenect2 */