New VideoToolbox rgb packet processor

Mac OS X >= 10.8 has hardware accelerated jpeg decoding (a bit hidden)

Author: fran6co

CMakeLists.txt

@@ -53,7 +53,6 @@ SET(LIBRARY_OUTPUT_PATH ${PROJECT_BINARY_DIR}/lib)
 # dependencies
 FIND_PACKAGE(PkgConfig)    # try find PKGConfig as it will be used if found
 FIND_PACKAGE(LibUSB REQUIRED)
-FIND_PACKAGE(TurboJPEG REQUIRED) #does not provide a package-config file
 
 # Add includes
 INCLUDE_DIRECTORIES(
@@ -62,7 +61,6 @@ INCLUDE_DIRECTORIES(
   ${PROJECT_BINARY_DIR} # for generated headers
   ${LIBFREENECT2_THREADING_INCLUDE_DIR}
   ${LibUSB_INCLUDE_DIRS}
-  ${TurboJPEG_INCLUDE_DIRS}
 )
 
 LINK_DIRECTORIES(${LibUSB_LIBRARY_DIRS})
@@ -105,7 +103,6 @@ SET(SOURCES
   src/packet_pipeline.cpp
   src/rgb_packet_stream_parser.cpp
   src/rgb_packet_processor.cpp
-  src/turbo_jpeg_rgb_packet_processor.cpp
   src/depth_packet_stream_parser.cpp
   src/depth_packet_processor.cpp
   src/cpu_depth_packet_processor.cpp
@@ -122,10 +119,52 @@ SET(SOURCES
 
 SET(LIBRARIES
   ${LibUSB_LIBRARIES}
-  ${TurboJPEG_LIBRARIES}
   ${LIBFREENECT2_THREADING_LIBRARIES}
 )
 
+IF(APPLE)
+  FIND_LIBRARY(VIDEOTOOLBOX_LIBRARY VideoToolbox)
+
+  IF(VIDEOTOOLBOX_LIBRARY)
+    SET(LIBFREENECT2_WITH_VT_SUPPORT 1)
+
+    FIND_LIBRARY(COREFOUNDATION_LIBRARY CoreFoundation REQUIRED)
+    FIND_LIBRARY(COREMEDIA_LIBRARY CoreMedia REQUIRED)
+    FIND_LIBRARY(COREVIDEO_LIBRARY CoreVideo REQUIRED)
+
+    LIST(APPEND SOURCES
+      src/vt_rgb_packet_processor.cpp
+    )
+
+    LIST(APPEND LIBRARIES
+      ${VIDEOTOOLBOX_LIBRARY}
+      ${COREFOUNDATION_LIBRARY}
+      ${COREMEDIA_LIBRARY}
+      ${COREVIDEO_LIBRARY}
+    )
+  ENDIF(VIDEOTOOLBOX_LIBRARY)
+ENDIF(APPLE)
+
+IF(LIBFREENECT2_WITH_VT_SUPPORT)
+  FIND_PACKAGE(TurboJPEG)
+ELSE()
+  FIND_PACKAGE(TurboJPEG REQUIRED)
+ENDIF()
+
+IF(TurboJPEG_FOUND)
+  SET(LIBFREENECT2_WITH_TURBOJPEG_SUPPORT 1)
+
+  INCLUDE_DIRECTORIES(${TurboJPEG_INCLUDE_DIRS})
+
+  LIST(APPEND SOURCES
+    src/turbo_jpeg_rgb_packet_processor.cpp
+  )
+
+  LIST(APPEND LIBRARIES
+    ${TurboJPEG_LIBRARIES}
+  )
+ENDIF()
+
 IF(ENABLE_OPENGL)
   FIND_PACKAGE(GLFW3)
   FIND_PACKAGE(OpenGL)

README.md

@@ -149,20 +149,16 @@ Use your favorite package managers (brew, ports, etc.)
 
 1. ``cd`` into a directory where you want to keep libfreenect2 stuff in
 1. Make sure these build tools are available: wget, git, cmake, pkg-config, automake, autoconf, libtool. Xcode may provide some of them. Install the rest via package managers.
-1. Install dependencies: TurboJPEG, GLFW.
+1. Install dependencies: GLFW.
 
     ```
 brew update
-brew tap homebrew/science
-brew install jpeg-turbo
 brew tap homebrew/versions
 brew install glfw3
 ```
 
     Do not install libusb via package managers for libfreenect2. libfreenect2 includes an unreleased local version of libusb with USB3 specific patches. libfreenect2's libusb should still work fine in presence of a global version libusb.
 
-    It is not recommended to build TurboJPEG from source, which produces corrupted results on Mac OSX according to reports. Install TurboJPEG binary only from package managers.
-
 1. Download the libfreenect2 repository
 
     ```

include/libfreenect2/config.h.in

@@ -40,6 +40,10 @@
 
 #cmakedefine LIBFREENECT2_WITH_OPENCL_SUPPORT
 
+#cmakedefine LIBFREENECT2_WITH_VT_SUPPORT
+
+#cmakedefine LIBFREENECT2_WITH_TURBOJPEG_SUPPORT
+
 #cmakedefine LIBFREENECT2_THREADING_STDLIB
 
 #cmakedefine LIBFREENECT2_THREADING_TINYTHREAD

include/libfreenect2/frame_listener.hpp

@@ -55,22 +55,30 @@ class LIBFREENECT2_API Frame
   size_t bytes_per_pixel; ///< Number of bytes in a pixel.
   unsigned char* data;    ///< Data of the frame (aligned).
 
-  Frame(size_t width, size_t height, size_t bytes_per_pixel) :
+  Frame(size_t width, size_t height, size_t bytes_per_pixel, unsigned char* _data = NULL) :
     width(width),
     height(height),
-    bytes_per_pixel(bytes_per_pixel)
+    bytes_per_pixel(bytes_per_pixel),
+    rawdata(NULL),
+    data(_data)
   {
-    const size_t alignment = 64;
-    size_t space = width * height * bytes_per_pixel + alignment;
-    rawdata = new unsigned char[space];
-    uintptr_t ptr = reinterpret_cast<uintptr_t>(rawdata);
-    uintptr_t aligned = (ptr - 1u + alignment) & -alignment;
-    data = reinterpret_cast<unsigned char *>(aligned);
+    if (data == NULL)
+    {
+      const size_t alignment = 64;
+      size_t space = width * height * bytes_per_pixel + alignment;
+      rawdata = new unsigned char[space];
+      uintptr_t ptr = reinterpret_cast<uintptr_t>(rawdata);
+      uintptr_t aligned = (ptr - 1u + alignment) & -alignment;
+      data = reinterpret_cast<unsigned char *>(aligned);
+    }
   }
 
-  ~Frame()
+  virtual ~Frame()
   {
-    delete[] rawdata;
+    if (rawdata)
+    {
+      delete[] rawdata;
+    }
   }
 
   protected:

include/libfreenect2/rgb_packet_processor.h

@@ -73,6 +73,7 @@ class LIBFREENECT2_API DumpRgbPacketProcessor : public RgbPacketProcessor
   virtual void process(const libfreenect2::RgbPacket &packet);
 };
 
+#ifdef LIBFREENECT2_WITH_TURBOJPEG_SUPPORT
 class TurboJpegRgbPacketProcessorImpl;
 
 /** Processor to decode JPEG to image, using TurboJpeg. */
@@ -86,6 +87,22 @@ class LIBFREENECT2_API TurboJpegRgbPacketProcessor : public RgbPacketProcessor
 private:
   TurboJpegRgbPacketProcessorImpl *impl_; ///< Decoder implementation.
 };
+#endif
+
+#ifdef LIBFREENECT2_WITH_VT_SUPPORT
+class VTRgbPacketProcessorImpl;
+
+class LIBFREENECT2_API VTRgbPacketProcessor : public RgbPacketProcessor
+{
+public:
+  VTRgbPacketProcessor();
+  virtual ~VTRgbPacketProcessor();
+protected:
+  virtual void process(const libfreenect2::RgbPacket &packet);
+private:
+  VTRgbPacketProcessorImpl *impl_;
+};
+#endif
 
 } /* namespace libfreenect2 */
 #endif /* RGB_PACKET_PROCESSOR_H_ */

src/packet_pipeline.cpp

@@ -44,7 +44,12 @@ void BasePacketPipeline::initialize()
   rgb_parser_ = new RgbPacketStreamParser();
   depth_parser_ = new DepthPacketStreamParser();
 
+#ifdef LIBFREENECT2_WITH_VT_SUPPORT
+  rgb_processor_ = new VTRgbPacketProcessor();
+#else
   rgb_processor_ = new TurboJpegRgbPacketProcessor();
+#endif
+
   depth_processor_ = createDepthPacketProcessor();
 
   async_rgb_processor_ = new AsyncPacketProcessor<RgbPacket>(rgb_processor_);

src/vt_rgb_packet_processor.cpp

@@ -0,0 +1,178 @@
+/*
+ * This file is part of the OpenKinect Project. http://www.openkinect.org
+ *
+ * Copyright (c) 2014 individual OpenKinect contributors. See the CONTRIB file
+ * for details.
+ *
+ * This code is licensed to you under the terms of the Apache License, version
+ * 2.0, or, at your option, the terms of the GNU General Public License,
+ * version 2.0. See the APACHE20 and GPL2 files for the text of the licenses,
+ * or the following URLs:
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * http://www.gnu.org/licenses/gpl-2.0.txt
+ *
+ * If you redistribute this file in source form, modified or unmodified, you
+ * may:
+ *   1) Leave this header intact and distribute it under the same terms,
+ *      accompanying it with the APACHE20 and GPL20 files, or
+ *   2) Delete the Apache 2.0 clause and accompany it with the GPL2 file, or
+ *   3) Delete the GPL v2 clause and accompany it with the APACHE20 file
+ * In all cases you must keep the copyright notice intact and include a copy
+ * of the CONTRIB file.
+ *
+ * Binary distributions must follow the binary distribution requirements of
+ * either License.
+ */
+
+
+#include <libfreenect2/rgb_packet_processor.h>
+#include <libfreenect2/logging.h>
+
+#include <VideoToolbox/VideoToolbox.h>
+
+namespace libfreenect2 {
+
+class VTFrame: public Frame
+{
+ public:
+  VTFrame(size_t width, size_t height, size_t bytes_per_pixel, CVPixelBufferRef pixelBuffer) :
+      Frame(width,
+            height,
+            bytes_per_pixel,
+            reinterpret_cast<unsigned char *>(CVPixelBufferGetBaseAddress(lockPixelBuffer(pixelBuffer)))),
+      pixelBuffer(pixelBuffer) {
+  }
+
+  ~VTFrame() {
+    CVPixelBufferUnlockBaseAddress(pixelBuffer, 0);
+    CVPixelBufferRelease(pixelBuffer);
+  }
+
+ protected:
+  CVPixelBufferRef lockPixelBuffer(CVPixelBufferRef _pixelBuffer) {
+    CVPixelBufferLockBaseAddress(_pixelBuffer, 0);
+
+    return _pixelBuffer;
+  }
+
+  CVPixelBufferRef pixelBuffer;
+};
+
+class VTRgbPacketProcessorImpl: public WithPerfLogging
+{
+ public:
+  CMFormatDescriptionRef format;
+  VTDecompressionSessionRef decoder;
+
+  VTRgbPacketProcessorImpl() {
+    int32_t width = 1920, height = 1080;
+
+    CMVideoFormatDescriptionCreate(NULL, kCMVideoCodecType_JPEG, width, height, nil, &format);
+
+    const void *decoderSpecificationKeys[] = {kVTVideoDecoderSpecification_EnableHardwareAcceleratedVideoDecoder};
+    const void *decoderSpecificationValues[] = {kCFBooleanTrue};
+    CFDictionaryRef decoderSpecification = CFDictionaryCreate(NULL,
+                                                              decoderSpecificationKeys,
+                                                              decoderSpecificationValues,
+                                                              1,
+                                                              &kCFTypeDictionaryKeyCallBacks,
+                                                              &kCFTypeDictionaryValueCallBacks);
+
+    int32_t pixelFormat = kCVPixelFormatType_32BGRA;
+    const void *outputKeys[] = {kCVPixelBufferPixelFormatTypeKey, kCVPixelBufferWidthKey, kCVPixelBufferHeightKey};
+    const void *outputValues[] =
+        {CFNumberCreate(NULL, kCFNumberSInt32Type, &pixelFormat), CFNumberCreate(NULL, kCFNumberSInt32Type, &width),
+         CFNumberCreate(NULL, kCFNumberSInt32Type, &height)};
+
+    CFDictionaryRef outputConfiguration = CFDictionaryCreate(NULL,
+                                                             outputKeys,
+                                                             outputValues,
+                                                             3,
+                                                             &kCFTypeDictionaryKeyCallBacks,
+                                                             &kCFTypeDictionaryValueCallBacks);
+
+    VTDecompressionOutputCallbackRecord callback = {&VTRgbPacketProcessorImpl::decodeFrame, NULL};
+
+    VTDecompressionSessionCreate(NULL, format, decoderSpecification, outputConfiguration, &callback, &decoder);
+
+    CFRelease(decoderSpecification);
+    CFRelease(outputConfiguration);
+  }
+
+  ~VTRgbPacketProcessorImpl() {
+    VTDecompressionSessionInvalidate(decoder);
+    CFRelease(decoder);
+    CFRelease(format);
+  }
+
+  static void decodeFrame(void *decompressionOutputRefCon,
+                          void *sourceFrameRefCon,
+                          OSStatus status,
+                          VTDecodeInfoFlags infoFlags,
+                          CVImageBufferRef pixelBuffer,
+                          CMTime presentationTimeStamp,
+                          CMTime presentationDuration) {
+    CVPixelBufferRef *outputPixelBuffer = (CVPixelBufferRef *) sourceFrameRefCon;
+    *outputPixelBuffer = CVPixelBufferRetain(pixelBuffer);
+  }
+};
+
+VTRgbPacketProcessor::VTRgbPacketProcessor() :
+    impl_(new VTRgbPacketProcessorImpl())
+{
+}
+
+VTRgbPacketProcessor::~VTRgbPacketProcessor()
+{
+  delete impl_;
+}
+
+void VTRgbPacketProcessor::process(const RgbPacket &packet)
+{
+  if (listener_ != 0) {
+    impl_->startTiming();
+
+    CMBlockBufferRef blockBuffer;
+    CMBlockBufferCreateWithMemoryBlock(
+        NULL,
+        packet.jpeg_buffer,
+        packet.jpeg_buffer_length,
+        kCFAllocatorNull,
+        NULL,
+        0,
+        packet.jpeg_buffer_length,
+        0,
+        &blockBuffer
+    );
+
+    CMSampleBufferRef sampleBuffer;
+    CMSampleBufferCreateReady(
+        NULL,
+        blockBuffer,
+        impl_->format,
+        1,
+        0,
+        NULL,
+        0,
+        NULL,
+        &sampleBuffer
+    );
+
+    CVPixelBufferRef pixelBuffer = NULL;
+    VTDecompressionSessionDecodeFrame(impl_->decoder, sampleBuffer, 0, &pixelBuffer, NULL);
+
+    Frame *frame = new VTFrame(1920, 1080, 4, pixelBuffer);
+
+    frame->timestamp = packet.timestamp;
+    frame->sequence = packet.sequence;
+
+    listener_->onNewFrame(Frame::Color, frame);
+
+    CFRelease(sampleBuffer);
+    CFRelease(blockBuffer);
+
+    impl_->stopTiming(LOG_INFO);
+  }
+}
+
+} /* namespace libfreenect2 */