Ignore non image sample buffers for streaming

Author: RobertOdrowaz

packages/camera/camera_avfoundation/CHANGELOG.md

@@ -1,3 +1,7 @@
+## 0.9.21
+
+* Fixes crash when streaming is enabled during recording.
+
 ## 0.9.20+7
 
 * Updates kotlin version to 2.2.0 to enable gradle 8.11 support.

packages/camera/camera_avfoundation/example/ios/RunnerTests/StreamingTests.swift

@@ -36,6 +36,7 @@ final class StreamingTests: XCTestCase {
     DefaultCamera,
     AVCaptureOutput,
     CMSampleBuffer,
+    CMSampleBuffer,
     AVCaptureConnection
   ) {
     let captureSessionQueue = DispatchQueue(label: "testing")
@@ -45,13 +46,14 @@ final class StreamingTests: XCTestCase {
     let camera = CameraTestUtils.createTestCamera(configuration)
     let testAudioOutput = CameraTestUtils.createTestAudioOutput()
     let sampleBuffer = CameraTestUtils.createTestSampleBuffer()
+    let audioSampleBuffer = CameraTestUtils.createTestAudioSampleBuffer()
     let testAudioConnection = CameraTestUtils.createTestConnection(testAudioOutput)
 
-    return (camera, testAudioOutput, sampleBuffer, testAudioConnection)
+    return (camera, testAudioOutput, sampleBuffer, audioSampleBuffer, testAudioConnection)
   }
 
   func testExceedMaxStreamingPendingFramesCount() {
-    let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
+    let (camera, testAudioOutput, sampleBuffer, _, testAudioConnection) = createCamera()
     let handlerMock = MockImageStreamHandler()
 
     let finishStartStreamExpectation = expectation(
@@ -87,7 +89,7 @@ final class StreamingTests: XCTestCase {
   }
 
   func testReceivedImageStreamData() {
-    let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
+    let (camera, testAudioOutput, sampleBuffer, _, testAudioConnection) = createCamera()
     let handlerMock = MockImageStreamHandler()
 
     let finishStartStreamExpectation = expectation(
@@ -124,8 +126,34 @@ final class StreamingTests: XCTestCase {
     waitForExpectations(timeout: 30, handler: nil)
   }
 
+  func testIgnoresNonImageBuffers() {
+    let (camera, testAudioOutput, _, audioSampleBuffer, testAudioConnection) = createCamera()
+    let handlerMock = MockImageStreamHandler()
+    handlerMock.eventSinkStub = { event in
+      XCTFail()
+    }
+
+    let finishStartStreamExpectation = expectation(
+      description: "Finish startStream")
+
+    let messenger = MockFlutterBinaryMessenger()
+    camera.startImageStream(
+      with: messenger, imageStreamHandler: handlerMock,
+      completion: {
+        _ in
+        finishStartStreamExpectation.fulfill()
+      })
+
+    waitForExpectations(timeout: 30, handler: nil)
+    XCTAssertEqual(camera.isStreamingImages, true)
+
+    camera.captureOutput(testAudioOutput, didOutput: audioSampleBuffer, from: testAudioConnection)
+
+    waitForQueueRoundTrip(with: DispatchQueue.main)
+  }
+
   func testImageStreamEventFormat() {
-    let (camera, testAudioOutput, sampleBuffer, testAudioConnection) = createCamera()
+    let (camera, testAudioOutput, sampleBuffer, _, testAudioConnection) = createCamera()
 
     let expectation = expectation(description: "Received a valid event")
 

packages/camera/camera_avfoundation/ios/camera_avfoundation/Sources/camera_avfoundation/DefaultCamera.swift

@@ -762,66 +762,67 @@ final class DefaultCamera: FLTCam, Camera {
       if let eventSink = imageStreamHandler?.eventSink,
         streamingPendingFramesCount < maxStreamingPendingFramesCount
       {
-        streamingPendingFramesCount += 1
-
-        let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer)!
-        // Must lock base address before accessing the pixel data
-        CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
-
-        let imageWidth = CVPixelBufferGetWidth(pixelBuffer)
-        let imageHeight = CVPixelBufferGetHeight(pixelBuffer)
-
-        var planes: [[String: Any]] = []
-
-        let isPlanar = CVPixelBufferIsPlanar(pixelBuffer)
-        let planeCount = isPlanar ? CVPixelBufferGetPlaneCount(pixelBuffer) : 1
-
-        for i in 0..<planeCount {
-          let planeAddress: UnsafeMutableRawPointer?
-          let bytesPerRow: Int
-          let height: Int
-          let width: Int
-
-          if isPlanar {
-            planeAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i)
-            bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i)
-            height = CVPixelBufferGetHeightOfPlane(pixelBuffer, i)
-            width = CVPixelBufferGetWidthOfPlane(pixelBuffer, i)
-          } else {
-            planeAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
-            bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
-            height = CVPixelBufferGetHeight(pixelBuffer)
-            width = CVPixelBufferGetWidth(pixelBuffer)
+        if let pixelBuffer = CMSampleBufferGetImageBuffer(sampleBuffer) {
+          streamingPendingFramesCount += 1
+
+          // Must lock base address before accessing the pixel data
+          CVPixelBufferLockBaseAddress(pixelBuffer, .readOnly)
+
+          let imageWidth = CVPixelBufferGetWidth(pixelBuffer)
+          let imageHeight = CVPixelBufferGetHeight(pixelBuffer)
+
+          var planes: [[String: Any]] = []
+
+          let isPlanar = CVPixelBufferIsPlanar(pixelBuffer)
+          let planeCount = isPlanar ? CVPixelBufferGetPlaneCount(pixelBuffer) : 1
+
+          for i in 0..<planeCount {
+            let planeAddress: UnsafeMutableRawPointer?
+            let bytesPerRow: Int
+            let height: Int
+            let width: Int
+
+            if isPlanar {
+              planeAddress = CVPixelBufferGetBaseAddressOfPlane(pixelBuffer, i)
+              bytesPerRow = CVPixelBufferGetBytesPerRowOfPlane(pixelBuffer, i)
+              height = CVPixelBufferGetHeightOfPlane(pixelBuffer, i)
+              width = CVPixelBufferGetWidthOfPlane(pixelBuffer, i)
+            } else {
+              planeAddress = CVPixelBufferGetBaseAddress(pixelBuffer)
+              bytesPerRow = CVPixelBufferGetBytesPerRow(pixelBuffer)
+              height = CVPixelBufferGetHeight(pixelBuffer)
+              width = CVPixelBufferGetWidth(pixelBuffer)
+            }
+
+            let length = bytesPerRow * height
+            let bytes = Data(bytes: planeAddress!, count: length)
+
+            let planeBuffer: [String: Any] = [
+              "bytesPerRow": bytesPerRow,
+              "width": width,
+              "height": height,
+              "bytes": FlutterStandardTypedData(bytes: bytes),
+            ]
+            planes.append(planeBuffer)
           }
 
-          let length = bytesPerRow * height
-          let bytes = Data(bytes: planeAddress!, count: length)
-
-          let planeBuffer: [String: Any] = [
-            "bytesPerRow": bytesPerRow,
-            "width": width,
-            "height": height,
-            "bytes": FlutterStandardTypedData(bytes: bytes),
+          // Lock the base address before accessing pixel data, and unlock it afterwards.
+          // Done accessing the `pixelBuffer` at this point.
+          CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
+
+          let imageBuffer: [String: Any] = [
+            "width": imageWidth,
+            "height": imageHeight,
+            "format": videoFormat,
+            "planes": planes,
+            "lensAperture": Double(captureDevice.lensAperture()),
+            "sensorExposureTime": Int(captureDevice.exposureDuration().seconds * 1_000_000_000),
+            "sensorSensitivity": Double(captureDevice.iso()),
           ]
-          planes.append(planeBuffer)
-        }
 
-        // Lock the base address before accessing pixel data, and unlock it afterwards.
-        // Done accessing the `pixelBuffer` at this point.
-        CVPixelBufferUnlockBaseAddress(pixelBuffer, .readOnly)
-
-        let imageBuffer: [String: Any] = [
-          "width": imageWidth,
-          "height": imageHeight,
-          "format": videoFormat,
-          "planes": planes,
-          "lensAperture": Double(captureDevice.lensAperture()),
-          "sensorExposureTime": Int(captureDevice.exposureDuration().seconds * 1_000_000_000),
-          "sensorSensitivity": Double(captureDevice.iso()),
-        ]
-
-        DispatchQueue.main.async {
-          eventSink(imageBuffer)
+          DispatchQueue.main.async {
+            eventSink(imageBuffer)
+          }
         }
       }
     }

packages/camera/camera_avfoundation/pubspec.yaml

@@ -2,7 +2,7 @@ name: camera_avfoundation
 description: iOS implementation of the camera plugin.
 repository: https://github.com/flutter/packages/tree/main/packages/camera/camera_avfoundation
 issue_tracker: https://github.com/flutter/flutter/issues?q=is%3Aissue+is%3Aopen+label%3A%22p%3A+camera%22
-version: 0.9.20+7
+version: 0.9.21
 
 environment:
   sdk: ^3.6.0