fix/screenshot masking during changes (#2553)

* capture screenshots in sequence until they're stable

* improve timestamps

* fix tests & future handling

* example: debug log only in debug mode

* formatting

* min-version compat

* linter issue

* fix scheduler tests

* cleanup

* fix: scheduler test

* improve performance

* improve screenshot comparison performance

* comments

* screenshot list-equal fix & tests

* min-version failure

* rename retrier to stabilizer

* chore: changelog

* capture stable issue screenshots when masking is enabled

* fixes and tests

* better logging

* cocoa replay fix

* fix dart2js tests

* fix screenshot capture hanging

* fix integration test

* time out if we can't take a screenshot for events

* update ios app size metrics check

* fix: oom

Author: vaind

CHANGELOG.md

@@ -6,12 +6,16 @@
 
 - Add `beforeCapture` for View Hierarchy ([#2523](https://github.com/getsentry/sentry-dart/pull/2523))
   - View hierarchy calls are now debounced for 2 seconds.
-  
+
 ### Enhancements
 
 - Replay: improve iOS native interop performance ([#2530](https://github.com/getsentry/sentry-dart/pull/2530))
 - Replay: improve orientation change tracking accuracy on Android ([#2540](https://github.com/getsentry/sentry-dart/pull/2540))
 
+### Fixes
+
+- Replay: fix masking for frames captured during UI changes ([#2553](https://github.com/getsentry/sentry-dart/pull/2553))
+
 ### Dependencies
 
 - Bump Cocoa SDK from v8.42.0 to v8.43.0 ([#2542](https://github.com/getsentry/sentry-dart/pull/2542), [#2548](https://github.com/getsentry/sentry-dart/pull/2548))

flutter/example/devtools_options.yaml

@@ -0,0 +1,3 @@
+description: This file stores settings for Dart & Flutter DevTools.
+documentation: https://docs.flutter.dev/tools/devtools/extensions#configure-extension-enablement-states
+extensions:

flutter/example/integration_test/integration_test.dart

@@ -20,6 +20,8 @@ void main() {
   const fakeDsn = 'https://[email protected]/1234567';
 
   IntegrationTestWidgetsFlutterBinding.ensureInitialized();
+  IntegrationTestWidgetsFlutterBinding.instance.framePolicy =
+      LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
 
   tearDown(() async {
     await Sentry.close();

flutter/example/lib/main.dart

@@ -79,7 +79,7 @@ Future<void> setupSentry(
       // We can enable Sentry debug logging during development. This is likely
       // going to log too much for your app, but can be useful when figuring out
       // configuration issues, e.g. finding out why your events are not uploaded.
-      options.debug = true;
+      options.debug = kDebugMode;
       options.spotlight = Spotlight(enabled: true);
       options.enableTimeToFullDisplayTracing = true;
       options.enableMetrics = true;

flutter/lib/src/event_processor/screenshot_event_processor.dart

@@ -9,6 +9,7 @@ import '../screenshot/recorder.dart';
 import '../screenshot/recorder_config.dart';
 import 'package:flutter/widgets.dart' as widget;
 
+import '../screenshot/stabilizer.dart';
 import '../utils/debouncer.dart';
 
 class ScreenshotEventProcessor implements EventProcessor {
@@ -125,20 +126,37 @@ class ScreenshotEventProcessor implements EventProcessor {
   }
 
   @internal
-  Future<Uint8List?> createScreenshot() =>
-      _recorder.capture(_convertImageToUint8List);
-
-  Future<Uint8List?> _convertImageToUint8List(Screenshot screenshot) async {
-    final byteData =
-        await screenshot.image.toByteData(format: ImageByteFormat.png);
-
-    final bytes = byteData?.buffer.asUint8List();
-    if (bytes?.isNotEmpty == true) {
-      return bytes;
+  Future<Uint8List?> createScreenshot() async {
+    if (_options.experimental.privacyForScreenshots == null) {
+      return _recorder.capture((screenshot) =>
+          screenshot.pngData.then((v) => v.buffer.asUint8List()));
     } else {
-      _options.logger(
-          SentryLevel.debug, 'Screenshot is 0 bytes, not attaching the image.');
-      return null;
+      // If masking is enabled, we need to use [ScreenshotStabilizer].
+      final completer = Completer<Uint8List?>();
+      final stabilizer = ScreenshotStabilizer(
+        _recorder, _options,
+        (screenshot) async {
+          final pngData = await screenshot.pngData;
+          completer.complete(pngData.buffer.asUint8List());
+        },
+        // This limits the amount of time to take a stable masked screenshot.
+        maxTries: 5,
+        // We need to force the frame the frame or this could hang indefinitely.
+        frameSchedulingMode: FrameSchedulingMode.forced,
+      );
+      try {
+        unawaited(
+            stabilizer.capture(Duration.zero).onError(completer.completeError));
+        // DO NOT return completer.future directly - we need to dispose first.
+        return await completer.future.timeout(const Duration(seconds: 1),
+            onTimeout: () {
+          _options.logger(
+              SentryLevel.warning, 'Timed out taking a stable screenshot.');
+          return null;
+        });
+      } finally {
+        stabilizer.dispose();
+      }
     }
   }
 }

flutter/lib/src/native/cocoa/cocoa_replay_recorder.dart

@@ -0,0 +1,40 @@
+import 'dart:async';
+import 'dart:typed_data';
+
+import 'package:meta/meta.dart';
+
+import '../../../sentry_flutter.dart';
+import '../../replay/replay_recorder.dart';
+import '../../screenshot/recorder.dart';
+import '../../screenshot/recorder_config.dart';
+import '../../screenshot/stabilizer.dart';
+
+@internal
+class CocoaReplayRecorder {
+  final SentryFlutterOptions _options;
+  final ScreenshotRecorder _recorder;
+  late final ScreenshotStabilizer<void> _stabilizer;
+  var _completer = Completer<Uint8List?>();
+
+  CocoaReplayRecorder(this._options)
+      : _recorder =
+            ReplayScreenshotRecorder(ScreenshotRecorderConfig(), _options) {
+    _stabilizer = ScreenshotStabilizer(_recorder, _options, (screenshot) async {
+      final pngData = await screenshot.pngData;
+      _options.logger(
+          SentryLevel.debug,
+          'Replay: captured screenshot ('
+          '${screenshot.width}x${screenshot.height} pixels, '
+          '${pngData.lengthInBytes} bytes)');
+      _completer.complete(pngData.buffer.asUint8List());
+    });
+  }
+
+  Future<Uint8List?> captureScreenshot() async {
+    _completer = Completer<Uint8List?>();
+    _stabilizer.ensureFrameAndAddCallback((msSinceEpoch) {
+      _stabilizer.capture(msSinceEpoch).onError(_completer.completeError);
+    });
+    return _completer.future;
+  }
+}

flutter/lib/src/native/cocoa/sentry_native_cocoa.dart

@@ -1,23 +1,19 @@
 import 'dart:async';
 import 'dart:ffi';
-import 'dart:typed_data';
-import 'dart:ui';
 
 import 'package:meta/meta.dart';
 
 import '../../../sentry_flutter.dart';
 import '../../replay/replay_config.dart';
-import '../../replay/replay_recorder.dart';
-import '../../screenshot/recorder.dart';
-import '../../screenshot/recorder_config.dart';
 import '../native_memory.dart';
 import '../sentry_native_channel.dart';
 import 'binding.dart' as cocoa;
+import 'cocoa_replay_recorder.dart';
 
 @internal
 class SentryNativeCocoa extends SentryNativeChannel {
   late final _lib = cocoa.SentryCocoa(DynamicLibrary.process());
-  ScreenshotRecorder? _replayRecorder;
+  CocoaReplayRecorder? _replayRecorder;
   SentryId? _replayId;
 
   SentryNativeCocoa(super.options);
@@ -33,12 +29,12 @@ class SentryNativeCocoa extends SentryNativeChannel {
       channel.setMethodCallHandler((call) async {
         switch (call.method) {
           case 'captureReplayScreenshot':
-            _replayRecorder ??=
-                ReplayScreenshotRecorder(ScreenshotRecorderConfig(), options);
+            _replayRecorder ??= CocoaReplayRecorder(options);
 
             final replayId = call.arguments['replayId'] == null
                 ? null
                 : SentryId.fromId(call.arguments['replayId'] as String);
+
             if (_replayId != replayId) {
               _replayId = replayId;
               hub.configureScope((s) {
@@ -47,35 +43,7 @@ class SentryNativeCocoa extends SentryNativeChannel {
               });
             }
 
-            final widgetsBinding = options.bindingUtils.instance;
-            if (widgetsBinding == null) {
-              options.logger(SentryLevel.warning,
-                  'Replay: failed to capture screenshot, WidgetsBinding.instance is null');
-              return null;
-            }
-
-            final completer = Completer<Uint8List?>();
-            widgetsBinding.ensureVisualUpdate();
-            widgetsBinding.addPostFrameCallback((_) {
-              _replayRecorder?.capture((screenshot) async {
-                final image = screenshot.image;
-                final imageData =
-                    await image.toByteData(format: ImageByteFormat.png);
-                if (imageData != null) {
-                  options.logger(
-                      SentryLevel.debug,
-                      'Replay: captured screenshot ('
-                      '${image.width}x${image.height} pixels, '
-                      '${imageData.lengthInBytes} bytes)');
-                  return imageData.buffer.asUint8List();
-                } else {
-                  options.logger(SentryLevel.warning,
-                      'Replay: failed to convert screenshot to PNG');
-                }
-              }).then(completer.complete, onError: completer.completeError);
-            });
-            final uint8List = await completer.future;
-
+            final uint8List = await _replayRecorder!.captureScreenshot();
             // Malloc memory and copy the data. Native must free it.
             return uint8List?.toNativeMemory().toJson();
           default:

flutter/lib/src/native/java/android_replay_recorder.dart

@@ -2,6 +2,7 @@ import 'package:meta/meta.dart';
 
 import '../../../sentry_flutter.dart';
 import '../../replay/scheduled_recorder.dart';
+import '../../screenshot/screenshot.dart';
 import '../sentry_safe_method_channel.dart';
 
 @internal
@@ -15,19 +16,20 @@ class AndroidReplayRecorder extends ScheduledScreenshotRecorder {
   }
 
   Future<void> _addReplayScreenshot(
-      ScreenshotPng screenshot, bool isNewlyCaptured) async {
+      Screenshot screenshot, bool isNewlyCaptured) async {
     final timestamp = screenshot.timestamp.millisecondsSinceEpoch;
     final filePath = "$_cacheDir/$timestamp.png";
 
-    options.logger(
-        SentryLevel.debug,
-        '$logName: saving ${isNewlyCaptured ? 'new' : 'repeated'} screenshot to'
-        ' $filePath (${screenshot.width}x${screenshot.height} pixels, '
-        '${screenshot.data.lengthInBytes} bytes)');
     try {
+      final pngData = await screenshot.pngData;
+      options.logger(
+          SentryLevel.debug,
+          '$logName: saving ${isNewlyCaptured ? 'new' : 'repeated'} screenshot to'
+          ' $filePath (${screenshot.width}x${screenshot.height} pixels, '
+          '${pngData.lengthInBytes} bytes)');
       await options.fileSystem
           .file(filePath)
-          .writeAsBytes(screenshot.data.buffer.asUint8List(), flush: true);
+          .writeAsBytes(pngData.buffer.asUint8List(), flush: true);
 
       await _channel.invokeMethod(
         'addReplayScreenshot',

flutter/lib/src/replay/replay_recorder.dart

@@ -16,8 +16,8 @@ class ReplayScreenshotRecorder extends ScreenshotRecorder {
 
   @override
   @protected
-  Future<void> executeTask(void Function() task, Flow flow) {
-    // Future() schedules the task to be executed asynchronously with TImer.run.
+  Future<void> executeTask(Future<void> Function() task, Flow flow) {
+    // Future() schedules the task to be executed asynchronously with Timer.run.
     return Future(task);
   }
 }