diff --git a/CHANGELOG.md b/CHANGELOG.md index 5b906efab3..487e82e1f3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,7 +14,7 @@ ### Enhancements -- Replay: improve iOS native interop performance ([#2530](https://github.com/getsentry/sentry-dart/pull/2530)) +- Replay: improve iOS native interop performance ([#2530](https://github.com/getsentry/sentry-dart/pull/2530), [#2573](https://github.com/getsentry/sentry-dart/pull/2573)) - Replay: improve orientation change tracking accuracy on Android ([#2540](https://github.com/getsentry/sentry-dart/pull/2540)) ### Fixes diff --git a/flutter/ios/Classes/SentryFlutterReplayScreenshotProvider.m b/flutter/ios/Classes/SentryFlutterReplayScreenshotProvider.m index 2abe56ac43..7a6d8f2422 100644 --- a/flutter/ios/Classes/SentryFlutterReplayScreenshotProvider.m +++ b/flutter/ios/Classes/SentryFlutterReplayScreenshotProvider.m @@ -33,11 +33,45 @@ - (void)imageWithView:(UIView *_Nonnull)view NSDictionary *dict = (NSDictionary *)value; long address = ((NSNumber *)dict[@"address"]).longValue; NSNumber *length = ((NSNumber *)dict[@"length"]); + NSNumber *width = ((NSNumber *)dict[@"width"]); + NSNumber *height = ((NSNumber *)dict[@"height"]); NSData *data = [NSData dataWithBytesNoCopy:(void *)address length:length.unsignedLongValue freeWhenDone:TRUE]; - UIImage *image = [UIImage imageWithData:data]; + + // We expect rawRGBA, see docs for ImageByteFormat: + // https://api.flutter.dev/flutter/dart-ui/ImageByteFormat.html + // Unencoded bytes, in RGBA row-primary form with premultiplied + // alpha, 8 bits per channel. + static const int kBitsPerChannel = 8; + static const int kBytesPerPixel = 4; + assert(length.unsignedLongValue % kBytesPerPixel == 0); + + // Let's create an UIImage from the raw data. + // We need to provide it the width & height and + // the info how the data is encoded. + CGDataProviderRef provider = + CGDataProviderCreateWithCFData((CFDataRef)data); + CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB(); + CGBitmapInfo bitmapInfo = + kCGBitmapByteOrderDefault | kCGImageAlphaPremultipliedLast; + CGImageRef cgImage = CGImageCreate( + width.unsignedLongValue, // width + height.unsignedLongValue, // height + kBitsPerChannel, // bits per component + kBitsPerChannel * kBytesPerPixel, // bits per pixel + width.unsignedLongValue * kBytesPerPixel, // bytes per row + colorSpace, bitmapInfo, provider, NULL, false, + kCGRenderingIntentDefault); + + UIImage *image = [UIImage imageWithCGImage:cgImage]; + + // UIImage takes its own refs, we need to release these here. + CGImageRelease(cgImage); + CGColorSpaceRelease(colorSpace); + CGDataProviderRelease(provider); + onComplete(image); return; } else if ([value isKindOfClass:[FlutterError class]]) { diff --git a/flutter/lib/src/native/cocoa/cocoa_replay_recorder.dart b/flutter/lib/src/native/cocoa/cocoa_replay_recorder.dart index 8404dfaa9f..a9b0f34de8 100644 --- a/flutter/lib/src/native/cocoa/cocoa_replay_recorder.dart +++ b/flutter/lib/src/native/cocoa/cocoa_replay_recorder.dart @@ -1,5 +1,4 @@ import 'dart:async'; -import 'dart:typed_data'; import 'package:meta/meta.dart'; @@ -8,30 +7,36 @@ import '../../replay/replay_recorder.dart'; import '../../screenshot/recorder.dart'; import '../../screenshot/recorder_config.dart'; import '../../screenshot/stabilizer.dart'; +import '../native_memory.dart'; @internal class CocoaReplayRecorder { final SentryFlutterOptions _options; final ScreenshotRecorder _recorder; late final ScreenshotStabilizer _stabilizer; - var _completer = Completer(); + var _completer = Completer?>(); CocoaReplayRecorder(this._options) : _recorder = ReplayScreenshotRecorder(ScreenshotRecorderConfig(), _options) { _stabilizer = ScreenshotStabilizer(_recorder, _options, (screenshot) async { - final pngData = await screenshot.pngData; + final data = await screenshot.rawRgbaData; _options.logger( SentryLevel.debug, 'Replay: captured screenshot (' '${screenshot.width}x${screenshot.height} pixels, ' - '${pngData.lengthInBytes} bytes)'); - _completer.complete(pngData.buffer.asUint8List()); + '${data.lengthInBytes} bytes)'); + + // Malloc memory and copy the data. Native must free it. + final json = data.toNativeMemory().toJson(); + json['width'] = screenshot.width; + json['height'] = screenshot.height; + _completer.complete(json); }); } - Future captureScreenshot() async { - _completer = Completer(); + Future?> captureScreenshot() async { + _completer = Completer(); _stabilizer.ensureFrameAndAddCallback((msSinceEpoch) { _stabilizer.capture(msSinceEpoch).onError(_completer.completeError); }); diff --git a/flutter/lib/src/native/cocoa/sentry_native_cocoa.dart b/flutter/lib/src/native/cocoa/sentry_native_cocoa.dart index 658b5f8af0..56f7a4ef13 100644 --- a/flutter/lib/src/native/cocoa/sentry_native_cocoa.dart +++ b/flutter/lib/src/native/cocoa/sentry_native_cocoa.dart @@ -5,7 +5,6 @@ import 'package:meta/meta.dart'; import '../../../sentry_flutter.dart'; import '../../replay/replay_config.dart'; -import '../native_memory.dart'; import '../sentry_native_channel.dart'; import 'binding.dart' as cocoa; import 'cocoa_replay_recorder.dart'; @@ -43,9 +42,7 @@ class SentryNativeCocoa extends SentryNativeChannel { }); } - final uint8List = await _replayRecorder!.captureScreenshot(); - // Malloc memory and copy the data. Native must free it. - return uint8List?.toNativeMemory().toJson(); + return _replayRecorder!.captureScreenshot(); default: throw UnimplementedError('Method ${call.method} not implemented'); } diff --git a/flutter/lib/src/native/native_memory.dart b/flutter/lib/src/native/native_memory.dart index a7c7be0008..3cbe4698e9 100644 --- a/flutter/lib/src/native/native_memory.dart +++ b/flutter/lib/src/native/native_memory.dart @@ -12,13 +12,24 @@ class NativeMemory { const NativeMemory._(this.pointer, this.length); - factory NativeMemory.fromUint8List(Uint8List source) { - final length = source.length; - final ptr = pkg_ffi.malloc.allocate(length); - if (length > 0) { - ptr.asTypedList(length).setAll(0, source); + factory NativeMemory.fromByteData(ByteData source) { + final lengthInBytes = source.lengthInBytes; + final ptr = pkg_ffi.malloc.allocate(lengthInBytes); + + // TODO memcpy() from source.buffer.asUint8List().address + // once we can depend on Dart SDK 3.5+ + final numWords = lengthInBytes ~/ 8; + final words = ptr.cast().asTypedList(numWords); + if (numWords > 0) { + words.setAll(0, source.buffer.asUint64List(0, numWords)); } - return NativeMemory._(ptr, length); + + final bytes = ptr.asTypedList(lengthInBytes); + for (var i = words.lengthInBytes; i < source.lengthInBytes; i++) { + bytes[i] = source.getUint8(i); + } + + return NativeMemory._(ptr, lengthInBytes); } factory NativeMemory.fromJson(Map json) { @@ -44,6 +55,6 @@ class NativeMemory { } @internal -extension Uint8ListNativeMemory on Uint8List { - NativeMemory toNativeMemory() => NativeMemory.fromUint8List(this); +extension ByteDataNativeMemory on ByteData { + NativeMemory toNativeMemory() => NativeMemory.fromByteData(this); } diff --git a/flutter/lib/src/screenshot/screenshot.dart b/flutter/lib/src/screenshot/screenshot.dart index 4abfd45637..a2008fa8a0 100644 --- a/flutter/lib/src/screenshot/screenshot.dart +++ b/flutter/lib/src/screenshot/screenshot.dart @@ -12,20 +12,20 @@ class Screenshot { final Image _image; final DateTime timestamp; final Flow flow; - Future? _rawData; + Future? _rawRgbaData; Future? _pngData; bool _disposed = false; Screenshot(this._image, this.timestamp, this.flow); Screenshot._cloned( - this._image, this.timestamp, this.flow, this._rawData, this._pngData); + this._image, this.timestamp, this.flow, this._rawRgbaData, this._pngData); int get width => _image.width; int get height => _image.height; - Future get rawData { - _rawData ??= _encode(ImageByteFormat.rawUnmodified); - return _rawData!; + Future get rawRgbaData { + _rawRgbaData ??= _encode(ImageByteFormat.rawRgba); + return _rawRgbaData!; } Future get pngData { @@ -46,20 +46,20 @@ class Screenshot { return false; } - return listEquals(await rawData, await other.rawData); + return listEquals(await rawRgbaData, await other.rawRgbaData); } Screenshot clone() { assert(!_disposed, 'Cannot clone a disposed screenshot'); return Screenshot._cloned( - _image.clone(), timestamp, flow, _rawData, _pngData); + _image.clone(), timestamp, flow, _rawRgbaData, _pngData); } void dispose() { if (!_disposed) { _disposed = true; _image.dispose(); - _rawData = null; + _rawRgbaData = null; _pngData = null; } } diff --git a/flutter/test/native_memory_test.dart b/flutter/test/native_memory_test.dart index 055ccf90f7..185590ac69 100644 --- a/flutter/test/native_memory_test.dart +++ b/flutter/test/native_memory_test.dart @@ -8,10 +8,11 @@ import 'native_memory_web_mock.dart' if (dart.library.io) 'package:sentry_flutter/src/native/native_memory.dart'; void main() { - final testSrcList = Uint8List.fromList([1, 2, 3]); + final testSrcList = Uint8List.fromList([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]); + final testSrcData = testSrcList.buffer.asByteData(); test('empty list', () async { - final sut = NativeMemory.fromUint8List(Uint8List.fromList([])); + final sut = NativeMemory.fromByteData(ByteData(0)); expect(sut.length, 0); expect(sut.pointer.address, greaterThan(0)); expect(sut.asTypedList(), isEmpty); @@ -19,18 +20,18 @@ void main() { }); test('non-empty list', () async { - final sut = NativeMemory.fromUint8List(testSrcList); - expect(sut.length, 3); + final sut = NativeMemory.fromByteData(testSrcData); + expect(sut.length, 10); expect(sut.pointer.address, greaterThan(0)); expect(sut.asTypedList(), testSrcList); sut.free(); }); test('json', () async { - final sut = NativeMemory.fromUint8List(testSrcList); + final sut = NativeMemory.fromByteData(testSrcData); final json = sut.toJson(); expect(json['address'], greaterThan(0)); - expect(json['length'], 3); + expect(json['length'], 10); expect(json.entries, hasLength(2)); final sut2 = NativeMemory.fromJson(json); diff --git a/flutter/test/native_memory_web_mock.dart b/flutter/test/native_memory_web_mock.dart index 81e03fee6f..820532c84c 100644 --- a/flutter/test/native_memory_web_mock.dart +++ b/flutter/test/native_memory_web_mock.dart @@ -9,8 +9,9 @@ class NativeMemory { const NativeMemory._(this.pointer, this.length); - factory NativeMemory.fromUint8List(Uint8List source) { - return NativeMemory._(Pointer._store(source), source.length); + factory NativeMemory.fromByteData(ByteData source) { + return NativeMemory._(Pointer._store(source.buffer.asUint8List()), + source.lengthInBytes); } factory NativeMemory.fromJson(Map json) { diff --git a/flutter/test/replay/replay_native_test.dart b/flutter/test/replay/replay_native_test.dart index 983f4274c6..baa5493e70 100644 --- a/flutter/test/replay/replay_native_test.dart +++ b/flutter/test/replay/replay_native_test.dart @@ -209,6 +209,8 @@ void main() { expect(json['length'], greaterThan(3000)); expect(json['address'], greaterThan(0)); + expect(json['width'], greaterThan(0)); + expect(json['height'], greaterThan(0)); NativeMemory.fromJson(json).free(); }