diff --git a/sentry-android-core/api/sentry-android-core.api b/sentry-android-core/api/sentry-android-core.api index 51eb48f1b22..55278c63563 100644 --- a/sentry-android-core/api/sentry-android-core.api +++ b/sentry-android-core/api/sentry-android-core.api @@ -245,6 +245,10 @@ public final class io/sentry/android/core/SentryAndroid { public static fun init (Landroid/content/Context;Lio/sentry/ILogger;)V public static fun init (Landroid/content/Context;Lio/sentry/ILogger;Lio/sentry/Sentry$OptionsConfiguration;)V public static fun init (Landroid/content/Context;Lio/sentry/Sentry$OptionsConfiguration;)V + public static fun pauseReplay ()V + public static fun resumeReplay ()V + public static fun startReplay ()V + public static fun stopReplay ()V } public final class io/sentry/android/core/SentryAndroidDateProvider : io/sentry/SentryDateProvider { diff --git a/sentry-android-core/build.gradle.kts b/sentry-android-core/build.gradle.kts index 4ab0aec4231..6ea33c7b74e 100644 --- a/sentry-android-core/build.gradle.kts +++ b/sentry-android-core/build.gradle.kts @@ -76,6 +76,7 @@ dependencies { api(projects.sentry) compileOnly(projects.sentryAndroidFragment) compileOnly(projects.sentryAndroidTimber) + compileOnly(projects.sentryAndroidReplay) compileOnly(projects.sentryCompose) // lifecycle processor, session tracking @@ -103,6 +104,7 @@ dependencies { testImplementation(projects.sentryTestSupport) testImplementation(projects.sentryAndroidFragment) testImplementation(projects.sentryAndroidTimber) + testImplementation(projects.sentryAndroidReplay) testImplementation(projects.sentryComposeHelper) testImplementation(projects.sentryAndroidNdk) testRuntimeOnly(Config.Libs.composeUi) diff --git a/sentry-android-core/proguard-rules.pro b/sentry-android-core/proguard-rules.pro index 67d7e7691d5..a78a5a14a19 100644 --- a/sentry-android-core/proguard-rules.pro +++ b/sentry-android-core/proguard-rules.pro @@ -72,3 +72,9 @@ -keepnames class io.sentry.exception.SentryHttpClientException ##---------------End: proguard configuration for sentry-okhttp ---------- + +##---------------Begin: proguard configuration for sentry-android-replay ---------- +-dontwarn io.sentry.android.replay.ReplayIntegration +-dontwarn io.sentry.android.replay.ReplayIntegrationKt +-keepnames class io.sentry.android.replay.ReplayIntegration +##---------------End: proguard configuration for sentry-android-replay ---------- diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java index 41d0dec6b2b..b58051cee71 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/AndroidOptionsInitializer.java @@ -22,6 +22,7 @@ import io.sentry.android.core.internal.util.SentryFrameMetricsCollector; import io.sentry.android.core.performance.AppStartMetrics; import io.sentry.android.fragment.FragmentLifecycleIntegration; +import io.sentry.android.replay.ReplayIntegration; import io.sentry.android.timber.SentryTimberIntegration; import io.sentry.cache.PersistingOptionsObserver; import io.sentry.cache.PersistingScopeObserver; @@ -29,6 +30,7 @@ import io.sentry.compose.viewhierarchy.ComposeViewHierarchyExporter; import io.sentry.internal.gestures.GestureTargetLocator; import io.sentry.internal.viewhierarchy.ViewHierarchyExporter; +import io.sentry.transport.CurrentDateProvider; import io.sentry.transport.NoOpEnvelopeCache; import io.sentry.util.LazyEvaluator; import io.sentry.util.Objects; @@ -230,7 +232,8 @@ static void installDefaultIntegrations( final @NotNull LoadClass loadClass, final @NotNull ActivityFramesTracker activityFramesTracker, final boolean isFragmentAvailable, - final boolean isTimberAvailable) { + final boolean isTimberAvailable, + final boolean isReplayAvailable) { // Integration MUST NOT cache option values in ctor, as they will be configured later by the // user @@ -295,6 +298,9 @@ static void installDefaultIntegrations( new NetworkBreadcrumbsIntegration(context, buildInfoProvider, options.getLogger())); options.addIntegration(new TempSensorBreadcrumbsIntegration(context)); options.addIntegration(new PhoneStateBreadcrumbsIntegration(context)); + if (isReplayAvailable) { + options.addIntegration(new ReplayIntegration(context, CurrentDateProvider.getInstance())); + } } /** diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/LifecycleWatcher.java b/sentry-android-core/src/main/java/io/sentry/android/core/LifecycleWatcher.java index 7b38bcd9c2f..ff281d2beba 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/LifecycleWatcher.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/LifecycleWatcher.java @@ -11,6 +11,7 @@ import io.sentry.transport.ICurrentDateProvider; import java.util.Timer; import java.util.TimerTask; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; @@ -19,11 +20,12 @@ final class LifecycleWatcher implements DefaultLifecycleObserver { private final AtomicLong lastUpdatedSession = new AtomicLong(0L); + private final AtomicBoolean isFreshSession = new AtomicBoolean(false); private final long sessionIntervalMillis; private @Nullable TimerTask timerTask; - private final @Nullable Timer timer; + private final @NotNull Timer timer = new Timer(true); private final @NotNull Object timerLock = new Object(); private final @NotNull IHub hub; private final boolean enableSessionTracking; @@ -55,11 +57,6 @@ final class LifecycleWatcher implements DefaultLifecycleObserver { this.enableAppLifecycleBreadcrumbs = enableAppLifecycleBreadcrumbs; this.hub = hub; this.currentDateProvider = currentDateProvider; - if (enableSessionTracking) { - timer = new Timer(true); - } else { - timer = null; - } } // App goes to foreground @@ -74,41 +71,45 @@ public void onStart(final @NotNull LifecycleOwner owner) { } private void startSession() { - if (enableSessionTracking) { - cancelTask(); + cancelTask(); - final long currentTimeMillis = currentDateProvider.getCurrentTimeMillis(); + final long currentTimeMillis = currentDateProvider.getCurrentTimeMillis(); - hub.configureScope( - scope -> { - if (lastUpdatedSession.get() == 0L) { - final @Nullable Session currentSession = scope.getSession(); - if (currentSession != null && currentSession.getStarted() != null) { - lastUpdatedSession.set(currentSession.getStarted().getTime()); - } + hub.configureScope( + scope -> { + if (lastUpdatedSession.get() == 0L) { + final @Nullable Session currentSession = scope.getSession(); + if (currentSession != null && currentSession.getStarted() != null) { + lastUpdatedSession.set(currentSession.getStarted().getTime()); + isFreshSession.set(true); } - }); + } + }); - final long lastUpdatedSession = this.lastUpdatedSession.get(); - if (lastUpdatedSession == 0L - || (lastUpdatedSession + sessionIntervalMillis) <= currentTimeMillis) { + final long lastUpdatedSession = this.lastUpdatedSession.get(); + if (lastUpdatedSession == 0L + || (lastUpdatedSession + sessionIntervalMillis) <= currentTimeMillis) { + if (enableSessionTracking) { addSessionBreadcrumb("start"); hub.startSession(); } - this.lastUpdatedSession.set(currentTimeMillis); + SentryAndroid.startReplay(); + } else if (!isFreshSession.getAndSet(false)) { + // only resume if it's not a fresh session, which has been started in SentryAndroid.init + SentryAndroid.resumeReplay(); } + this.lastUpdatedSession.set(currentTimeMillis); } // App went to background and triggered this callback after 700ms // as no new screen was shown @Override public void onStop(final @NotNull LifecycleOwner owner) { - if (enableSessionTracking) { - final long currentTimeMillis = currentDateProvider.getCurrentTimeMillis(); - this.lastUpdatedSession.set(currentTimeMillis); + final long currentTimeMillis = currentDateProvider.getCurrentTimeMillis(); + this.lastUpdatedSession.set(currentTimeMillis); - scheduleEndSession(); - } + SentryAndroid.pauseReplay(); + scheduleEndSession(); AppState.getInstance().setInBackground(true); addAppBreadcrumb("background"); @@ -122,8 +123,11 @@ private void scheduleEndSession() { new TimerTask() { @Override public void run() { - addSessionBreadcrumb("end"); - hub.endSession(); + if (enableSessionTracking) { + addSessionBreadcrumb("end"); + hub.endSession(); + } + SentryAndroid.stopReplay(); } }; @@ -164,7 +168,7 @@ TimerTask getTimerTask() { } @TestOnly - @Nullable + @NotNull Timer getTimer() { return timer; } diff --git a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java index af68a026fbb..d6e11d15f54 100644 --- a/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java +++ b/sentry-android-core/src/main/java/io/sentry/android/core/SentryAndroid.java @@ -15,6 +15,8 @@ import io.sentry.android.core.performance.AppStartMetrics; import io.sentry.android.core.performance.TimeSpan; import io.sentry.android.fragment.FragmentLifecycleIntegration; +import io.sentry.android.replay.ReplayIntegration; +import io.sentry.android.replay.ReplayIntegrationKt; import io.sentry.android.timber.SentryTimberIntegration; import java.lang.reflect.InvocationTargetException; import java.util.ArrayList; @@ -33,6 +35,11 @@ public final class SentryAndroid { static final String SENTRY_TIMBER_INTEGRATION_CLASS_NAME = "io.sentry.android.timber.SentryTimberIntegration"; + static final String SENTRY_REPLAY_INTEGRATION_CLASS_NAME = + "io.sentry.android.replay.ReplayIntegration"; + + private static boolean isReplayAvailable = false; + private static final String TIMBER_CLASS_NAME = "timber.log.Timber"; private static final String FRAGMENT_CLASS_NAME = "androidx.fragment.app.FragmentManager$FragmentLifecycleCallbacks"; @@ -99,6 +106,8 @@ public static synchronized void init( final boolean isTimberAvailable = (isTimberUpstreamAvailable && classLoader.isClassAvailable(SENTRY_TIMBER_INTEGRATION_CLASS_NAME, options)); + isReplayAvailable = + classLoader.isClassAvailable(SENTRY_REPLAY_INTEGRATION_CLASS_NAME, options); final BuildInfoProvider buildInfoProvider = new BuildInfoProvider(logger); final LoadClass loadClass = new LoadClass(); @@ -118,7 +127,8 @@ public static synchronized void init( loadClass, activityFramesTracker, isFragmentAvailable, - isTimberAvailable); + isTimberAvailable, + isReplayAvailable); configuration.configure(options); @@ -145,9 +155,12 @@ public static synchronized void init( true); final @NotNull IHub hub = Sentry.getCurrentHub(); - if (hub.getOptions().isEnableAutoSessionTracking() && ContextUtils.isForegroundImportance()) { - hub.addBreadcrumb(BreadcrumbFactory.forSession("session.start")); - hub.startSession(); + if (ContextUtils.isForegroundImportance()) { + if (hub.getOptions().isEnableAutoSessionTracking()) { + hub.addBreadcrumb(BreadcrumbFactory.forSession("session.start")); + hub.startSession(); + } + startReplay(); } } catch (IllegalAccessException e) { logger.log(SentryLevel.FATAL, "Fatal error during SentryAndroid.init(...)", e); @@ -212,4 +225,59 @@ private static void deduplicateIntegrations( } } } + + public static synchronized void startReplay() { + if (!ensureReplayIntegration("starting")) { + return; + } + final @NotNull IHub hub = Sentry.getCurrentHub(); + ReplayIntegrationKt.getReplayIntegration(hub).start(); + } + + public static synchronized void stopReplay() { + if (!ensureReplayIntegration("stopping")) { + return; + } + final @NotNull IHub hub = Sentry.getCurrentHub(); + ReplayIntegrationKt.getReplayIntegration(hub).stop(); + } + + public static synchronized void resumeReplay() { + if (!ensureReplayIntegration("resuming")) { + return; + } + final @NotNull IHub hub = Sentry.getCurrentHub(); + ReplayIntegrationKt.getReplayIntegration(hub).resume(); + } + + public static synchronized void pauseReplay() { + if (!ensureReplayIntegration("pausing")) { + return; + } + final @NotNull IHub hub = Sentry.getCurrentHub(); + ReplayIntegrationKt.getReplayIntegration(hub).pause(); + } + + private static boolean ensureReplayIntegration(final @NotNull String actionName) { + final @NotNull IHub hub = Sentry.getCurrentHub(); + if (isReplayAvailable) { + final ReplayIntegration replay = ReplayIntegrationKt.getReplayIntegration(hub); + if (replay != null) { + return true; + } else { + hub.getOptions() + .getLogger() + .log( + SentryLevel.INFO, + "Session Replay wasn't registered yet, not " + actionName + " the replay"); + } + } else { + hub.getOptions() + .getLogger() + .log( + SentryLevel.INFO, + "Session Replay wasn't found on classpath, not " + actionName + " the replay"); + } + return false; + } } diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt index 6353e9dde89..94b0490f17d 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidOptionsInitializerTest.kt @@ -15,6 +15,7 @@ import io.sentry.android.core.internal.gestures.AndroidViewGestureTargetLocator import io.sentry.android.core.internal.modules.AssetsModulesLoader import io.sentry.android.core.internal.util.AndroidMainThreadChecker import io.sentry.android.fragment.FragmentLifecycleIntegration +import io.sentry.android.replay.ReplayIntegration import io.sentry.android.timber.SentryTimberIntegration import io.sentry.cache.PersistingOptionsObserver import io.sentry.cache.PersistingScopeObserver @@ -83,6 +84,7 @@ class AndroidOptionsInitializerTest { loadClass, activityFramesTracker, false, + false, false ) @@ -99,7 +101,8 @@ class AndroidOptionsInitializerTest { minApi: Int = Build.VERSION_CODES.KITKAT, classesToLoad: List = emptyList(), isFragmentAvailable: Boolean = false, - isTimberAvailable: Boolean = false + isTimberAvailable: Boolean = false, + isReplayAvailable: Boolean = false ) { mockContext = ContextUtilsTestHelper.mockMetaData( mockContext = ContextUtilsTestHelper.createMockContext(hasAppContext = true), @@ -126,7 +129,8 @@ class AndroidOptionsInitializerTest { loadClass, activityFramesTracker, isFragmentAvailable, - isTimberAvailable + isTimberAvailable, + isReplayAvailable ) AndroidOptionsInitializer.initializeIntegrationsAndProcessors( @@ -478,6 +482,24 @@ class AndroidOptionsInitializerTest { assertNull(actual) } + @Test + fun `ReplayIntegration added to the integration list if available on classpath`() { + fixture.initSutWithClassLoader(isReplayAvailable = true) + + val actual = + fixture.sentryOptions.integrations.firstOrNull { it is ReplayIntegration } + assertNotNull(actual) + } + + @Test + fun `ReplayIntegration won't be enabled, it throws class not found`() { + fixture.initSutWithClassLoader(isReplayAvailable = false) + + val actual = + fixture.sentryOptions.integrations.firstOrNull { it is ReplayIntegration } + assertNull(actual) + } + @Test fun `AndroidEnvelopeCache is set to options`() { fixture.initSut() @@ -634,6 +656,7 @@ class AndroidOptionsInitializerTest { mock(), mock(), false, + false, false ) verify(mockOptions, never()).outboxPath diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidProfilerTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidProfilerTest.kt index a726b2c55b8..2efb6020755 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidProfilerTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidProfilerTest.kt @@ -118,6 +118,7 @@ class AndroidProfilerTest { loadClass, activityFramesTracker, false, + false, false ) diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidTransactionProfilerTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidTransactionProfilerTest.kt index 405aa6dc98b..9376ea79fb3 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/AndroidTransactionProfilerTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/AndroidTransactionProfilerTest.kt @@ -125,6 +125,7 @@ class AndroidTransactionProfilerTest { loadClass, activityFramesTracker, false, + false, false ) diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/LifecycleWatcherTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/LifecycleWatcherTest.kt index be309931429..4b620813bf5 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/LifecycleWatcherTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/LifecycleWatcherTest.kt @@ -123,7 +123,6 @@ class LifecycleWatcherTest { fun `When session tracking is disabled, do not end session`() { val watcher = fixture.getSUT(enableAutoSessionTracking = false, enableAppLifecycleBreadcrumbs = false) watcher.onStop(fixture.ownerMock) - assertNull(watcher.timerTask) verify(fixture.hub, never()).endSession() } @@ -167,7 +166,6 @@ class LifecycleWatcherTest { fun `When session tracking is disabled, do not add breadcrumb on stop`() { val watcher = fixture.getSUT(enableAutoSessionTracking = false, enableAppLifecycleBreadcrumbs = false) watcher.onStop(fixture.ownerMock) - assertNull(watcher.timerTask) verify(fixture.hub, never()).addBreadcrumb(any()) } @@ -219,12 +217,6 @@ class LifecycleWatcherTest { assertNotNull(watcher.timer) } - @Test - fun `timer is not created if session tracking is disabled`() { - val watcher = fixture.getSUT(enableAutoSessionTracking = false, enableAppLifecycleBreadcrumbs = false) - assertNull(watcher.timer) - } - @Test fun `if the hub has already a fresh session running, don't start new one`() { val watcher = fixture.getSUT( diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt index bd5b3695fb2..b543ae318a4 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/SentryAndroidTest.kt @@ -26,6 +26,8 @@ import io.sentry.UncaughtExceptionHandlerIntegration import io.sentry.android.core.cache.AndroidEnvelopeCache import io.sentry.android.core.performance.AppStartMetrics import io.sentry.android.fragment.FragmentLifecycleIntegration +import io.sentry.android.replay.ReplayIntegration +import io.sentry.android.replay.getReplayIntegration import io.sentry.android.timber.SentryTimberIntegration import io.sentry.cache.IEnvelopeCache import io.sentry.cache.PersistingOptionsObserver @@ -313,12 +315,26 @@ class SentryAndroidTest { } } + @Test + @Config(sdk = [26]) + fun `init starts session replay if app is in foreground`() { + initSentryWithForegroundImportance(true) { _ -> + assertTrue(Sentry.getCurrentHub().getReplayIntegration()!!.isRecording()) + } + } + + @Test + @Config(sdk = [26]) + fun `init does not start session replay if the app is in background`() { + initSentryWithForegroundImportance(false) { _ -> + assertFalse(Sentry.getCurrentHub().getReplayIntegration()!!.isRecording()) + } + } + private fun initSentryWithForegroundImportance( inForeground: Boolean, callback: (session: Session?) -> Unit ) { - val context = ContextUtilsTestHelper.createMockContext() - Mockito.mockStatic(ContextUtils::class.java).use { mockedContextUtils -> mockedContextUtils.`when` { ContextUtils.isForegroundImportance() } .thenReturn(inForeground) @@ -412,7 +428,7 @@ class SentryAndroidTest { fixture.initSut(context = mock()) { options -> optionsRef = options options.dsn = "https://key@sentry.io/123" - assertEquals(19, options.integrations.size) + assertEquals(20, options.integrations.size) options.integrations.removeAll { it is UncaughtExceptionHandlerIntegration || it is ShutdownHookIntegration || @@ -431,7 +447,8 @@ class SentryAndroidTest { it is SystemEventsBreadcrumbsIntegration || it is NetworkBreadcrumbsIntegration || it is TempSensorBreadcrumbsIntegration || - it is PhoneStateBreadcrumbsIntegration + it is PhoneStateBreadcrumbsIntegration || + it is ReplayIntegration } } assertEquals(0, optionsRef.integrations.size) diff --git a/sentry-android-core/src/test/java/io/sentry/android/core/SentryInitProviderTest.kt b/sentry-android-core/src/test/java/io/sentry/android/core/SentryInitProviderTest.kt index a83076efb04..5b546523d01 100644 --- a/sentry-android-core/src/test/java/io/sentry/android/core/SentryInitProviderTest.kt +++ b/sentry-android-core/src/test/java/io/sentry/android/core/SentryInitProviderTest.kt @@ -143,6 +143,7 @@ class SentryInitProviderTest { loadClass, activityFramesTracker, false, + false, false ) diff --git a/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro b/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro index 49f7f0749d8..02f5e80ba30 100644 --- a/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro +++ b/sentry-android-integration-tests/sentry-uitest-android/proguard-rules.pro @@ -39,4 +39,4 @@ -dontwarn org.opentest4j.AssertionFailedError -dontwarn org.mockito.internal.** -dontwarn org.jetbrains.annotations.** - +-dontwarn io.sentry.android.replay.ReplayIntegration diff --git a/sentry-android-replay/api/sentry-android-replay.api b/sentry-android-replay/api/sentry-android-replay.api index e81c5840eac..7f45c6d8f73 100644 --- a/sentry-android-replay/api/sentry-android-replay.api +++ b/sentry-android-replay/api/sentry-android-replay.api @@ -6,10 +6,72 @@ public final class io/sentry/android/replay/BuildConfig { public fun ()V } -public final class io/sentry/android/replay/WindowRecorder { - public fun ()V - public final fun startRecording (Landroid/content/Context;)V - public final fun stopRecording ()V +public final class io/sentry/android/replay/GeneratedVideo { + public fun (Ljava/io/File;IJ)V + public final fun component1 ()Ljava/io/File; + public final fun component2 ()I + public final fun component3 ()J + public final fun copy (Ljava/io/File;IJ)Lio/sentry/android/replay/GeneratedVideo; + public static synthetic fun copy$default (Lio/sentry/android/replay/GeneratedVideo;Ljava/io/File;IJILjava/lang/Object;)Lio/sentry/android/replay/GeneratedVideo; + public fun equals (Ljava/lang/Object;)Z + public final fun getDuration ()J + public final fun getFrameCount ()I + public final fun getVideo ()Ljava/io/File; + public fun hashCode ()I + public fun toString ()Ljava/lang/String; +} + +public final class io/sentry/android/replay/ReplayCache : java/io/Closeable { + public fun (Lio/sentry/SentryOptions;Lio/sentry/protocol/SentryId;Lio/sentry/android/replay/ScreenshotRecorderConfig;)V + public final fun addFrame (Ljava/io/File;J)V + public fun close ()V + public final fun createVideoOf (JJILjava/io/File;)Lio/sentry/android/replay/GeneratedVideo; + public static synthetic fun createVideoOf$default (Lio/sentry/android/replay/ReplayCache;JJILjava/io/File;ILjava/lang/Object;)Lio/sentry/android/replay/GeneratedVideo; +} + +public final class io/sentry/android/replay/ReplayIntegration : io/sentry/Integration, io/sentry/android/replay/ScreenshotRecorderCallback, java/io/Closeable { + public static final field Companion Lio/sentry/android/replay/ReplayIntegration$Companion; + public static final field VIDEO_BUFFER_DURATION J + public static final field VIDEO_SEGMENT_DURATION J + public fun (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V + public fun close ()V + public final fun isRecording ()Z + public fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V + public final fun pause ()V + public fun register (Lio/sentry/IHub;Lio/sentry/SentryOptions;)V + public final fun resume ()V + public final fun start ()V + public final fun stop ()V +} + +public final class io/sentry/android/replay/ReplayIntegration$Companion { +} + +public final class io/sentry/android/replay/ReplayIntegrationKt { + public static final fun getReplayIntegration (Lio/sentry/IHub;)Lio/sentry/android/replay/ReplayIntegration; + public static final fun gracefullyShutdown (Ljava/util/concurrent/ExecutorService;Lio/sentry/SentryOptions;)V +} + +public abstract interface class io/sentry/android/replay/ScreenshotRecorderCallback { + public abstract fun onScreenshotRecorded (Landroid/graphics/Bitmap;)V +} + +public final class io/sentry/android/replay/ScreenshotRecorderConfig { + public fun (IIFI)V + public synthetic fun (IIFIILkotlin/jvm/internal/DefaultConstructorMarker;)V + public final fun component1 ()I + public final fun component2 ()I + public final fun component3 ()F + public final fun component4 ()I + public final fun copy (IIFI)Lio/sentry/android/replay/ScreenshotRecorderConfig; + public static synthetic fun copy$default (Lio/sentry/android/replay/ScreenshotRecorderConfig;IIFIILjava/lang/Object;)Lio/sentry/android/replay/ScreenshotRecorderConfig; + public fun equals (Ljava/lang/Object;)Z + public final fun getFrameRate ()I + public final fun getRecordingHeight ()I + public final fun getRecordingWidth ()I + public final fun getScaleFactor ()F + public fun hashCode ()I + public fun toString ()Ljava/lang/String; } public abstract interface class io/sentry/android/replay/video/SimpleFrameMuxer { diff --git a/sentry-android-replay/build.gradle.kts b/sentry-android-replay/build.gradle.kts index 2314960f5e2..319386ee2b7 100644 --- a/sentry-android-replay/build.gradle.kts +++ b/sentry-android-replay/build.gradle.kts @@ -19,6 +19,8 @@ android { targetSdk = Config.Android.targetSdkVersion minSdk = Config.Android.minSdkVersionReplay + testInstrumentationRunner = Config.TestLibs.androidJUnitRunner + // for AGP 4.1 buildConfigField("String", "VERSION_NAME", "\"${project.version}\"") } @@ -67,7 +69,9 @@ dependencies { // tests testImplementation(projects.sentryTestSupport) + testImplementation(Config.TestLibs.robolectric) testImplementation(Config.TestLibs.kotlinTestJunit) + testImplementation(Config.TestLibs.androidxRunner) testImplementation(Config.TestLibs.androidxJunit) testImplementation(Config.TestLibs.mockitoKotlin) testImplementation(Config.TestLibs.mockitoInline) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt new file mode 100644 index 00000000000..fd07d74354a --- /dev/null +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayCache.kt @@ -0,0 +1,236 @@ +package io.sentry.android.replay + +import android.graphics.Bitmap +import android.graphics.Bitmap.CompressFormat.JPEG +import android.graphics.BitmapFactory +import io.sentry.SentryLevel.DEBUG +import io.sentry.SentryLevel.ERROR +import io.sentry.SentryLevel.WARNING +import io.sentry.SentryOptions +import io.sentry.android.replay.video.MuxerConfig +import io.sentry.android.replay.video.SimpleVideoEncoder +import io.sentry.protocol.SentryId +import java.io.Closeable +import java.io.File + +/** + * A basic in-memory and disk cache for Session Replay frames. Frames are stored in order under the + * [SentryOptions.cacheDirPath] + [replayId] folder. The class is also capable of creating an mp4 + * video segment out of the stored frames, provided start time and duration using the available + * on-device [android.media.MediaCodec]. + * + * This class is not thread-safe, meaning, [addFrame] cannot be called concurrently with + * [createVideoOf], and they should be invoked from the same thread. + * + * @param options SentryOptions instance, used for logging and cacheDir + * @param replayId the current replay id, used for giving a unique name to the replay folder + * @param recorderConfig ScreenshotRecorderConfig, used for video resolution and frame-rate + */ +public class ReplayCache internal constructor( + private val options: SentryOptions, + private val replayId: SentryId, + private val recorderConfig: ScreenshotRecorderConfig, + private val encoderCreator: (File) -> SimpleVideoEncoder +) : Closeable { + + public constructor( + options: SentryOptions, + replayId: SentryId, + recorderConfig: ScreenshotRecorderConfig + ) : this(options, replayId, recorderConfig, encoderCreator = { videoFile -> + SimpleVideoEncoder( + options, + MuxerConfig( + file = videoFile, + recorderConfig = recorderConfig, + frameRate = recorderConfig.frameRate.toFloat(), + bitrate = 20 * 1000 + ) + ).also { it.start() } + }) + + private val encoderLock = Any() + private var encoder: SimpleVideoEncoder? = null + + internal val replayCacheDir: File? by lazy { + if (options.cacheDirPath.isNullOrEmpty()) { + options.logger.log( + WARNING, + "SentryOptions.cacheDirPath is not set, session replay is no-op" + ) + null + } else { + File(options.cacheDirPath!!, "replay_$replayId").also { it.mkdirs() } + } + } + + // TODO: maybe account for multi-threaded access + internal val frames = mutableListOf() + + /** + * Stores the current frame screenshot to in-memory cache as well as disk with [frameTimestamp] + * as filename. Uses [Bitmap.CompressFormat.JPEG] format with quality 80. The frames are stored + * under [replayCacheDir]. + * + * This method is not thread-safe. + * + * @param bitmap the frame screenshot + * @param frameTimestamp the timestamp when the frame screenshot was taken + */ + internal fun addFrame(bitmap: Bitmap, frameTimestamp: Long) { + if (replayCacheDir == null) { + return + } + + val screenshot = File(replayCacheDir, "$frameTimestamp.jpg").also { + it.createNewFile() + } + screenshot.outputStream().use { + bitmap.compress(JPEG, 80, it) + it.flush() + } + + addFrame(screenshot, frameTimestamp) + } + + /** + * Same as [addFrame], but accepts frame screenshot as [File], the file should contain + * a bitmap/image by the time [createVideoOf] is invoked. + * + * This method is not thread-safe. + * + * @param screenshot file containing the frame screenshot + * @param frameTimestamp the timestamp when the frame screenshot was taken + */ + public fun addFrame(screenshot: File, frameTimestamp: Long) { + val frame = ReplayFrame(screenshot, frameTimestamp) + frames += frame + } + + /** + * Creates a video out of currently stored [frames] given the start time and duration using the + * on-device codecs [android.media.MediaCodec]. The generated video will be stored in + * [videoFile] location, which defaults to "[replayCacheDir]/[segmentId].mp4". + * + * This method is not thread-safe. + * + * @param duration desired video duration in milliseconds + * @param from desired start of the video represented as unix timestamp in milliseconds + * @param segmentId current segment id, used for inferring the filename to store the + * result video under [replayCacheDir], e.g. "replay_/0.mp4", where segmentId=0 + * @param videoFile optional, location of the file to store the result video. If this is + * provided, [segmentId] from above is disregarded and not used. + * @return a generated video of type [GeneratedVideo], which contains the resulting video file + * location, frame count and duration in milliseconds. + */ + public fun createVideoOf( + duration: Long, + from: Long, + segmentId: Int, + videoFile: File = File(replayCacheDir, "$segmentId.mp4") + ): GeneratedVideo? { + if (frames.isEmpty()) { + options.logger.log( + DEBUG, + "No captured frames, skipping generating a video segment" + ) + return null + } + + // TODO: reuse instance of encoder and just change file path to create a different muxer + encoder = synchronized(encoderLock) { encoderCreator(videoFile) } + + val step = 1000 / recorderConfig.frameRate.toLong() + var frameCount = 0 + var lastFrame: ReplayFrame = frames.first() + for (timestamp in from until (from + (duration)) step step) { + val iter = frames.iterator() + while (iter.hasNext()) { + val frame = iter.next() + if (frame.timestamp in (timestamp..timestamp + step)) { + lastFrame = frame + break // we only support 1 frame per given interval + } + + // assuming frames are in order, if out of bounds exit early + if (frame.timestamp > timestamp + step) { + break + } + } + + // we either encode a new frame within the step bounds or replicate the last known frame + // to respect the video duration + if (encode(lastFrame)) { + frameCount++ + } + } + + if (frameCount == 0) { + options.logger.log( + DEBUG, + "Generated a video with no frames, not capturing a replay segment" + ) + deleteFile(videoFile) + return null + } + + var videoDuration: Long + synchronized(encoderLock) { + encoder?.release() + videoDuration = encoder?.duration ?: 0 + encoder = null + } + + frames.removeAll { + if (it.timestamp < (from + duration)) { + deleteFile(it.screenshot) + return@removeAll true + } + return@removeAll false + } + + return GeneratedVideo(videoFile, frameCount, videoDuration) + } + + private fun encode(frame: ReplayFrame): Boolean { + return try { + val bitmap = BitmapFactory.decodeFile(frame.screenshot.absolutePath) + synchronized(encoderLock) { + encoder?.encode(bitmap) + } + bitmap.recycle() + true + } catch (e: Throwable) { + options.logger.log(WARNING, "Unable to decode bitmap and encode it into a video, skipping frame", e) + false + } + } + + private fun deleteFile(file: File) { + try { + if (!file.delete()) { + options.logger.log(ERROR, "Failed to delete replay frame: %s", file.absolutePath) + } + } catch (e: Throwable) { + options.logger.log(ERROR, e, "Failed to delete replay frame: %s", file.absolutePath) + } + } + + override fun close() { + synchronized(encoderLock) { + encoder?.release() + encoder = null + } + } +} + +internal data class ReplayFrame( + val screenshot: File, + val timestamp: Long +) + +public data class GeneratedVideo( + val video: File, + val frameCount: Int, + val duration: Long +) diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt new file mode 100644 index 00000000000..e2244d0bf32 --- /dev/null +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt @@ -0,0 +1,320 @@ +package io.sentry.android.replay + +import android.content.Context +import android.graphics.Bitmap +import android.graphics.Point +import android.graphics.Rect +import android.os.Build +import android.os.Build.VERSION +import android.os.Build.VERSION_CODES +import android.view.WindowManager +import io.sentry.DateUtils +import io.sentry.Hint +import io.sentry.IHub +import io.sentry.Integration +import io.sentry.ReplayRecording +import io.sentry.SentryLevel.DEBUG +import io.sentry.SentryLevel.INFO +import io.sentry.SentryOptions +import io.sentry.SentryReplayEvent +import io.sentry.protocol.SentryId +import io.sentry.rrweb.RRWebMetaEvent +import io.sentry.rrweb.RRWebVideoEvent +import io.sentry.transport.ICurrentDateProvider +import io.sentry.util.FileUtils +import java.io.Closeable +import java.io.File +import java.util.Date +import java.util.concurrent.ExecutorService +import java.util.concurrent.Executors +import java.util.concurrent.ThreadFactory +import java.util.concurrent.TimeUnit.MILLISECONDS +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicInteger +import java.util.concurrent.atomic.AtomicReference +import kotlin.LazyThreadSafetyMode.NONE +import kotlin.math.roundToInt + +class ReplayIntegration( + private val context: Context, + private val dateProvider: ICurrentDateProvider +) : Integration, Closeable, ScreenshotRecorderCallback { + + companion object { + const val VIDEO_SEGMENT_DURATION = 5_000L + const val VIDEO_BUFFER_DURATION = 30_000L + } + + private lateinit var options: SentryOptions + private var hub: IHub? = null + private var recorder: WindowRecorder? = null + private var cache: ReplayCache? = null + + // TODO: probably not everything has to be thread-safe here + private val isEnabled = AtomicBoolean(false) + private val isRecording = AtomicBoolean(false) + private val currentReplayId = AtomicReference(SentryId.EMPTY_ID) + private val segmentTimestamp = AtomicReference() + private val currentSegment = AtomicInteger(0) + private val saver = + Executors.newSingleThreadScheduledExecutor(ReplayExecutorServiceThreadFactory()) + + private val screenBounds by lazy(NONE) { + // PixelCopy takes screenshots including system bars, so we have to get the real size here + val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager + if (VERSION.SDK_INT >= VERSION_CODES.R) { + wm.currentWindowMetrics.bounds + } else { + val screenBounds = Point() + @Suppress("DEPRECATION") + wm.defaultDisplay.getRealSize(screenBounds) + Rect(0, 0, screenBounds.x, screenBounds.y) + } + } + + private val aspectRatio by lazy(NONE) { + screenBounds.height().toFloat() / screenBounds.width().toFloat() + } + + private val recorderConfig by lazy(NONE) { + ScreenshotRecorderConfig( + recordingWidth = (720 / aspectRatio).roundToInt(), + recordingHeight = 720, + scaleFactor = 720f / screenBounds.bottom + ) + } + + override fun register(hub: IHub, options: SentryOptions) { + this.options = options + + if (Build.VERSION.SDK_INT < Build.VERSION_CODES.O) { + options.logger.log(INFO, "Session replay is only supported on API 26 and above") + return + } + + // TODO: check for replaysSessionSampleRate and replaysOnErrorSampleRate + + this.hub = hub + recorder = WindowRecorder(options, recorderConfig, this) + isEnabled.set(true) + } + + fun isRecording() = isRecording.get() + + fun start() { + // TODO: add lifecycle state instead and manage it in start/pause/resume/stop + if (!isEnabled.get()) { + options.logger.log( + DEBUG, + "Session replay is disabled due to conditions not met in Integration.register" + ) + return + } + + if (isRecording.getAndSet(true)) { + options.logger.log( + DEBUG, + "Session replay is already being recorded, not starting a new one" + ) + return + } + + currentSegment.set(0) + currentReplayId.set(SentryId()) + hub?.configureScope { it.replayId = currentReplayId.get() } + cache = ReplayCache(options, currentReplayId.get(), recorderConfig) + + recorder?.startRecording() + // TODO: replace it with dateProvider.currentTimeMillis to also test it + segmentTimestamp.set(DateUtils.getCurrentDateTime()) + // TODO: finalize old recording if there's some left on disk and send it using the replayId from persisted scope (e.g. for ANRs) + } + + fun resume() { + // TODO: replace it with dateProvider.currentTimeMillis to also test it + segmentTimestamp.set(DateUtils.getCurrentDateTime()) + recorder?.resume() + } + + fun pause() { + val now = dateProvider.currentTimeMillis + recorder?.pause() + + val currentSegmentTimestamp = segmentTimestamp.get() + val segmentId = currentSegment.get() + val duration = now - currentSegmentTimestamp.time + val replayId = currentReplayId.get() + saver.submit { + val videoDuration = + createAndCaptureSegment(duration, currentSegmentTimestamp, replayId, segmentId) + if (videoDuration != null) { + currentSegment.getAndIncrement() + } + } + } + + fun stop() { + if (!isEnabled.get()) { + options.logger.log( + DEBUG, + "Session replay is disabled due to conditions not met in Integration.register" + ) + return + } + + val now = dateProvider.currentTimeMillis + val currentSegmentTimestamp = segmentTimestamp.get() + val segmentId = currentSegment.get() + val duration = now - currentSegmentTimestamp.time + val replayId = currentReplayId.get() + val replayCacheDir = cache?.replayCacheDir + saver.submit { + createAndCaptureSegment(duration, currentSegmentTimestamp, replayId, segmentId) + FileUtils.deleteRecursively(replayCacheDir) + } + + recorder?.stopRecording() + cache?.close() + currentSegment.set(0) + segmentTimestamp.set(null) + currentReplayId.set(SentryId.EMPTY_ID) + hub?.configureScope { it.replayId = SentryId.EMPTY_ID } + isRecording.set(false) + } + + override fun onScreenshotRecorded(bitmap: Bitmap) { + // have to do it before submitting, otherwise if the queue is busy, the timestamp won't be + // reflecting the exact time of when it was captured + val frameTimestamp = dateProvider.currentTimeMillis + saver.submit { + cache?.addFrame(bitmap, frameTimestamp) + + val now = dateProvider.currentTimeMillis + if (now - segmentTimestamp.get().time >= VIDEO_SEGMENT_DURATION) { + val currentSegmentTimestamp = segmentTimestamp.get() + val segmentId = currentSegment.get() + val replayId = currentReplayId.get() + + val videoDuration = + createAndCaptureSegment( + VIDEO_SEGMENT_DURATION, + currentSegmentTimestamp, + replayId, + segmentId + ) + if (videoDuration != null) { + currentSegment.getAndIncrement() + // set next segment timestamp as close to the previous one as possible to avoid gaps + segmentTimestamp.set(DateUtils.getDateTime(currentSegmentTimestamp.time + videoDuration)) + } + } + } + } + + private fun createAndCaptureSegment( + duration: Long, + currentSegmentTimestamp: Date, + replayId: SentryId, + segmentId: Int + ): Long? { + val generatedVideo = cache?.createVideoOf( + duration, + currentSegmentTimestamp.time, + segmentId + ) ?: return null + + val (video, frameCount, videoDuration) = generatedVideo + captureReplay( + video, + replayId, + currentSegmentTimestamp, + segmentId, + frameCount, + videoDuration + ) + return videoDuration + } + + private fun captureReplay( + video: File, + currentReplayId: SentryId, + segmentTimestamp: Date, + segmentId: Int, + frameCount: Int, + duration: Long + ) { + val replay = SentryReplayEvent().apply { + eventId = currentReplayId + replayId = currentReplayId + this.segmentId = segmentId + this.timestamp = DateUtils.getDateTime(segmentTimestamp.time + duration) + if (segmentId == 0) { + replayStartTimestamp = segmentTimestamp + } + videoFile = video + } + + val recording = ReplayRecording().apply { + this.segmentId = segmentId + payload = listOf( + RRWebMetaEvent().apply { + this.timestamp = segmentTimestamp.time + height = recorderConfig.recordingHeight + width = recorderConfig.recordingWidth + }, + RRWebVideoEvent().apply { + this.timestamp = segmentTimestamp.time + this.segmentId = segmentId + this.durationMs = duration + this.frameCount = frameCount + size = video.length() + frameRate = recorderConfig.frameRate + height = recorderConfig.recordingHeight + width = recorderConfig.recordingWidth + // TODO: support non-fullscreen windows later + left = 0 + top = 0 + } + ) + } + + val hint = Hint().apply { replayRecording = recording } + hub?.captureReplay(replay, hint) + } + + override fun close() { + stop() + saver.gracefullyShutdown(options) + } + + private class ReplayExecutorServiceThreadFactory : ThreadFactory { + private var cnt = 0 + override fun newThread(r: Runnable): Thread { + val ret = Thread(r, "SentryReplayIntegration-" + cnt++) + ret.setDaemon(true) + return ret + } + } +} + +/** + * Retrieves the [ReplayIntegration] from the list of integrations in [SentryOptions] + */ +fun IHub.getReplayIntegration(): ReplayIntegration? = + options.integrations.find { it is ReplayIntegration } as? ReplayIntegration + +fun ExecutorService.gracefullyShutdown(options: SentryOptions) { + synchronized(this) { + if (!isShutdown) { + shutdown() + } + try { + if (!awaitTermination(options.shutdownTimeoutMillis, MILLISECONDS)) { + shutdownNow() + } + } catch (e: InterruptedException) { + shutdownNow() + Thread.currentThread().interrupt() + } + } +} diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt index 3b3a6758fc5..b403ceb4fa4 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt @@ -2,6 +2,7 @@ package io.sentry.android.replay import android.annotation.TargetApi import android.graphics.Bitmap +import android.graphics.Bitmap.Config.ARGB_8888 import android.graphics.Canvas import android.graphics.Matrix import android.graphics.Paint @@ -10,28 +11,31 @@ import android.graphics.RectF import android.os.Handler import android.os.HandlerThread import android.os.Looper -import android.os.SystemClock -import android.util.Log import android.view.PixelCopy import android.view.View import android.view.ViewGroup import android.view.ViewTreeObserver -import io.sentry.android.replay.video.SimpleVideoEncoder +import io.sentry.SentryLevel.DEBUG +import io.sentry.SentryLevel.INFO +import io.sentry.SentryLevel.WARNING +import io.sentry.SentryOptions import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode import java.lang.ref.WeakReference -import java.util.WeakHashMap +import java.util.concurrent.atomic.AtomicBoolean +import java.util.concurrent.atomic.AtomicReference import kotlin.system.measureTimeMillis -// TODO: use ILogger of Sentry and change level @TargetApi(26) internal class ScreenshotRecorder( - val encoder: SimpleVideoEncoder + val config: ScreenshotRecorderConfig, + val options: SentryOptions, + private val screenshotRecorderCallback: ScreenshotRecorderCallback ) : ViewTreeObserver.OnDrawListener { private var rootView: WeakReference? = null - private val thread = HandlerThread("SentryReplay").also { it.start() } + private val thread = HandlerThread("SentryReplayRecorder").also { it.start() } private val handler = Handler(thread.looper) - private val bitmapToVH = WeakHashMap() + private val pendingViewHierarchy = AtomicReference() private val maskingPaint = Paint() private val singlePixelBitmap: Bitmap = Bitmap.createBitmap( 1, @@ -40,104 +44,130 @@ internal class ScreenshotRecorder( ) private val singlePixelBitmapCanvas: Canvas = Canvas(singlePixelBitmap) private val prescaledMatrix = Matrix().apply { - preScale(encoder.muxerConfig.scaleFactor, encoder.muxerConfig.scaleFactor) + preScale(config.scaleFactor, config.scaleFactor) } + private val contentChanged = AtomicBoolean(false) + private val isCapturing = AtomicBoolean(true) + private var lastScreenshot: Bitmap? = null - companion object { - const val TAG = "ScreenshotRecorder" - } + fun capture() { + val viewHierarchy = pendingViewHierarchy.get() - private var lastCapturedAtMs: Long? = null - override fun onDraw() { - // TODO: replace with Debouncer from sentry-core - val now = SystemClock.uptimeMillis() - if (lastCapturedAtMs != null && (now - lastCapturedAtMs!!) < 500L) { + if (!isCapturing.get()) { + options.logger.log(DEBUG, "ScreenshotRecorder is paused, not capturing screenshot") + return + } + + if (!contentChanged.get() && lastScreenshot != null) { + options.logger.log(DEBUG, "Content hasn't changed, repeating last known frame") + + lastScreenshot?.let { + screenshotRecorderCallback.onScreenshotRecorded( + it.copy(ARGB_8888, false) + ) + } return } - lastCapturedAtMs = now val root = rootView?.get() if (root == null || root.width <= 0 || root.height <= 0 || !root.isShown) { + options.logger.log(DEBUG, "Root view is invalid, not capturing screenshot") + return + } + + val window = root.phoneWindow + if (window == null) { + options.logger.log(DEBUG, "Window is invalid, not capturing screenshot") return } - val window = root.phoneWindow ?: return val bitmap = Bitmap.createBitmap( root.width, root.height, Bitmap.Config.ARGB_8888 ) - val time = measureTimeMillis { - val rootNode = ViewHierarchyNode.fromView(root) - root.traverse(rootNode) - bitmapToVH[bitmap] = rootNode - } - Log.e("TIME", time.toString()) - // postAtFrontOfQueue to ensure the view hierarchy and bitmap are ase close in-sync as possible Handler(Looper.getMainLooper()).postAtFrontOfQueue { - PixelCopy.request( - window, - bitmap, - { copyResult: Int -> - Log.d(TAG, "PixelCopy result: $copyResult") - if (copyResult != PixelCopy.SUCCESS) { - Log.e(TAG, "Failed to capture screenshot") - return@request - } - - Log.e("BITMAP CAPTURED", bitmap.toString()) - val viewHierarchy = bitmapToVH[bitmap] - - var scaledBitmap: Bitmap? = null - - if (viewHierarchy == null) { - Log.e(TAG, "Failed to determine view hierarchy, not capturing") - return@request - } else { - scaledBitmap = Bitmap.createScaledBitmap( - bitmap, - encoder.muxerConfig.videoWidth, - encoder.muxerConfig.videoHeight, - true - ) - val canvas = Canvas(scaledBitmap) - canvas.setMatrix(prescaledMatrix) - viewHierarchy.traverse { - if (it.shouldRedact && (it.width > 0 && it.height > 0)) { - it.visibleRect ?: return@traverse - - // TODO: check for view type rather than rely on absence of dominantColor here - val color = if (it.dominantColor == null) { - singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null) - singlePixelBitmap.getPixel(0, 0) - } else { - it.dominantColor - } + try { + PixelCopy.request( + window, + bitmap, + { copyResult: Int -> + if (copyResult != PixelCopy.SUCCESS) { + options.logger.log(INFO, "Failed to capture replay recording: %d", copyResult) + bitmap.recycle() + return@request + } - maskingPaint.setColor(color) - canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint) + val scaledBitmap: Bitmap + + if (viewHierarchy == null) { + options.logger.log(INFO, "Failed to determine view hierarchy, not capturing") + bitmap.recycle() + return@request + } else { + scaledBitmap = Bitmap.createScaledBitmap( + bitmap, + config.recordingWidth, + config.recordingHeight, + true + ) + val canvas = Canvas(scaledBitmap) + canvas.setMatrix(prescaledMatrix) + viewHierarchy.traverse { + if (it.shouldRedact && (it.width > 0 && it.height > 0)) { + it.visibleRect ?: return@traverse + + // TODO: check for view type rather than rely on absence of dominantColor here + val color = if (it.dominantColor == null) { + singlePixelBitmapCanvas.drawBitmap(bitmap, it.visibleRect, Rect(0, 0, 1, 1), null) + singlePixelBitmap.getPixel(0, 0) + } else { + it.dominantColor + } + + maskingPaint.setColor(color) + canvas.drawRoundRect(RectF(it.visibleRect), 10f, 10f, maskingPaint) + } } } - } - -// val baos = ByteArrayOutputStream() -// scaledBitmap.compress(Bitmap.CompressFormat.JPEG, 75, baos) -// val bmp = BitmapFactory.decodeByteArray(baos.toByteArray(), 0, baos.size()) - scaledBitmap?.let { - encoder.encode(it) - it.recycle() - } -// bmp.recycle() - bitmap.recycle() - Log.i(TAG, "Captured a screenshot") - }, - handler - ) + + val screenshot = scaledBitmap.copy(ARGB_8888, false) + screenshotRecorderCallback.onScreenshotRecorded(screenshot) + lastScreenshot?.recycle() + lastScreenshot = screenshot + contentChanged.set(false) + + scaledBitmap.recycle() + bitmap.recycle() + }, + handler + ) + } catch (e: Throwable) { + options.logger.log(WARNING, "Failed to capture replay recording", e) + bitmap.recycle() + } } } + override fun onDraw() { + val root = rootView?.get() + if (root == null || root.width <= 0 || root.height <= 0 || !root.isShown) { + options.logger.log(DEBUG, "Root view is invalid, not capturing screenshot") + return + } + + val time = measureTimeMillis { + val rootNode = ViewHierarchyNode.fromView(root) + root.traverse(rootNode) + pendingViewHierarchy.set(rootNode) + } + options.logger.log(DEBUG, "Took %d ms to capture view hierarchy", time) + + contentChanged.set(true) + } + fun bind(root: View) { // first unbind the current root unbind(rootView?.get()) @@ -152,9 +182,23 @@ internal class ScreenshotRecorder( root?.viewTreeObserver?.removeOnDrawListener(this) } + fun pause() { + isCapturing.set(false) + unbind(rootView?.get()) + } + + fun resume() { + // can't use bind() as it will invalidate the weakref + rootView?.get()?.viewTreeObserver?.addOnDrawListener(this) + isCapturing.set(true) + } + fun close() { unbind(rootView?.get()) rootView?.clear() + lastScreenshot?.recycle() + pendingViewHierarchy.set(null) + isCapturing.set(false) thread.quitSafely() } @@ -188,3 +232,14 @@ internal class ScreenshotRecorder( parentNode.children = childNodes } } + +public data class ScreenshotRecorderConfig( + val recordingWidth: Int, + val recordingHeight: Int, + val scaleFactor: Float, + val frameRate: Int = 2 +) + +interface ScreenshotRecorderCallback { + fun onScreenshotRecorded(bitmap: Bitmap) +} diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt index 1a60d686b40..d23222368fe 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt @@ -1,31 +1,34 @@ package io.sentry.android.replay import android.annotation.TargetApi -import android.content.Context -import android.graphics.Point -import android.os.Build.VERSION -import android.os.Build.VERSION_CODES import android.view.View -import android.view.WindowManager -import io.sentry.android.replay.video.MuxerConfig -import io.sentry.android.replay.video.SimpleVideoEncoder -import java.io.File +import io.sentry.SentryLevel.ERROR +import io.sentry.SentryOptions +import java.io.Closeable import java.lang.ref.WeakReference +import java.util.concurrent.Executors +import java.util.concurrent.ScheduledFuture +import java.util.concurrent.ThreadFactory +import java.util.concurrent.TimeUnit.MILLISECONDS import java.util.concurrent.atomic.AtomicBoolean import kotlin.LazyThreadSafetyMode.NONE -import kotlin.math.roundToInt @TargetApi(26) -class WindowRecorder { +internal class WindowRecorder( + private val options: SentryOptions, + private val recorderConfig: ScreenshotRecorderConfig, + private val screenshotRecorderCallback: ScreenshotRecorderCallback +) : Closeable { private val rootViewsSpy by lazy(NONE) { RootViewsSpy.install() } - private var encoder: SimpleVideoEncoder? = null private val isRecording = AtomicBoolean(false) private val rootViews = ArrayList>() private var recorder: ScreenshotRecorder? = null + private var capturingTask: ScheduledFuture<*>? = null + private val capturer = Executors.newSingleThreadScheduledExecutor(RecorderExecutorServiceThreadFactory()) private val onRootViewsChangedListener = OnRootViewsChangedListener { root, added -> if (added) { @@ -42,53 +45,53 @@ class WindowRecorder { } } - fun startRecording(context: Context) { + fun startRecording() { if (isRecording.getAndSet(true)) { return } - val wm = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager // val (height, width) = (wm.currentWindowMetrics.bounds.bottom / // context.resources.displayMetrics.density).roundToInt() to // (wm.currentWindowMetrics.bounds.right / // context.resources.displayMetrics.density).roundToInt() - // TODO: API level check - // PixelCopy takes screenshots including system bars, so we have to get the real size here - val height: Int - val aspectRatio = if (VERSION.SDK_INT >= VERSION_CODES.R) { - height = wm.currentWindowMetrics.bounds.bottom - height.toFloat() / wm.currentWindowMetrics.bounds.right.toFloat() - } else { - val screenResolution = Point() - @Suppress("DEPRECATION") - wm.defaultDisplay.getRealSize(screenResolution) - height = screenResolution.y - height.toFloat() / screenResolution.x.toFloat() - } - val videoFile = File(context.cacheDir, "sentry-sr.mp4") - encoder = SimpleVideoEncoder( - MuxerConfig( - videoFile, - videoWidth = (720 / aspectRatio).roundToInt(), - videoHeight = 720, - scaleFactor = 720f / height, - frameRate = 2f, - bitrate = 500 * 1000 - ) - ).also { it.start() } - recorder = ScreenshotRecorder(encoder!!) + recorder = ScreenshotRecorder(recorderConfig, options, screenshotRecorderCallback) rootViewsSpy.listeners += onRootViewsChangedListener + capturingTask = capturer.scheduleAtFixedRate({ + try { + recorder?.capture() + } catch (e: Throwable) { + options.logger.log(ERROR, "Failed to capture a screenshot with exception:", e) + // TODO: I guess schedule the capturer again, cause it will stop executing the runnable? + } + }, 0L, 1000L / recorderConfig.frameRate, MILLISECONDS) } + fun resume() = recorder?.resume() + fun pause() = recorder?.pause() + fun stopRecording() { rootViewsSpy.listeners -= onRootViewsChangedListener rootViews.forEach { recorder?.unbind(it.get()) } recorder?.close() rootViews.clear() recorder = null - encoder?.startRelease() - encoder = null + capturingTask?.cancel(false) + capturingTask = null isRecording.set(false) } + + private class RecorderExecutorServiceThreadFactory : ThreadFactory { + private var cnt = 0 + override fun newThread(r: Runnable): Thread { + val ret = Thread(r, "SentryWindowRecorder-" + cnt++) + ret.setDaemon(true) + return ret + } + } + + override fun close() { + stopRecording() + capturer.gracefullyShutdown(options) + } } diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt index 86ff440d02c..e9c6761c75b 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/Windows.kt @@ -20,6 +20,8 @@ package io.sentry.android.replay import android.annotation.SuppressLint import android.os.Build.VERSION.SDK_INT +import android.os.Handler +import android.os.Looper import android.util.Log import android.view.View import android.view.Window @@ -137,6 +139,15 @@ internal class RootViewsSpy private constructor() { val listeners = CopyOnWriteArrayList() private val delegatingViewList = object : ArrayList() { + override fun addAll(elements: Collection): Boolean { + listeners.forEach { listener -> + elements.forEach { element -> + listener.onRootViewsChanged(element, true) + } + } + return super.addAll(elements) + } + override fun add(element: View): Boolean { listeners.forEach { it.onRootViewsChanged(element, true) } return super.add(element) @@ -152,8 +163,12 @@ internal class RootViewsSpy private constructor() { companion object { fun install(): RootViewsSpy { return RootViewsSpy().apply { - WindowManagerSpy.swapWindowManagerGlobalMViews { mViews -> - delegatingViewList.apply { addAll(mViews) } + // had to do this as a first message of the main thread queue, otherwise if this is + // called from ContentProvider, it might be too early and the listener won't be installed + Handler(Looper.getMainLooper()).postAtFrontOfQueue { + WindowManagerSpy.swapWindowManagerGlobalMViews { mViews -> + delegatingViewList.apply { addAll(mViews) } + } } } } diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt index bdedb888cdd..cf30f9e49fc 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleMp4FrameMuxer.kt @@ -32,12 +32,13 @@ package io.sentry.android.replay.video import android.media.MediaCodec import android.media.MediaFormat import android.media.MediaMuxer -import android.util.Log import java.nio.ByteBuffer import java.util.concurrent.TimeUnit +import java.util.concurrent.TimeUnit.MICROSECONDS +import java.util.concurrent.TimeUnit.MILLISECONDS -class SimpleMp4FrameMuxer(path: String, private val fps: Float) : SimpleFrameMuxer { - private val frameUsec: Long = (TimeUnit.SECONDS.toMicros(1L) / fps).toLong() +class SimpleMp4FrameMuxer(path: String, fps: Float) : SimpleFrameMuxer { + private val frameDurationUsec: Long = (TimeUnit.SECONDS.toMicros(1L) / fps).toLong() private val muxer: MediaMuxer = MediaMuxer(path, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4) @@ -50,7 +51,6 @@ class SimpleMp4FrameMuxer(path: String, private val fps: Float) : SimpleFrameMux override fun start(videoFormat: MediaFormat) { videoTrackIndex = muxer.addTrack(videoFormat) - Log.i("SimpleMp4FrameMuxer", "start() videoFormat=$videoFormat videoTrackIndex=$videoTrackIndex") muxer.start() started = true } @@ -59,7 +59,7 @@ class SimpleMp4FrameMuxer(path: String, private val fps: Float) : SimpleFrameMux // This code will break if the encoder supports B frames. // Ideally we would use set the value in the encoder, // don't know how to do that without using OpenGL - finalVideoTime = frameUsec * videoFrames++ + finalVideoTime = frameDurationUsec * videoFrames++ bufferInfo.presentationTimeUs = finalVideoTime // encodedData.position(bufferInfo.offset) @@ -74,6 +74,10 @@ class SimpleMp4FrameMuxer(path: String, private val fps: Float) : SimpleFrameMux } override fun getVideoTime(): Long { - return finalVideoTime + if (videoFrames == 0) { + return 0 + } + // have to add one sec as we calculate it 0-based above + return MILLISECONDS.convert(finalVideoTime + frameDurationUsec, MICROSECONDS) } } diff --git a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt index 4046eec37bc..e2561faa1bb 100644 --- a/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt +++ b/sentry-android-replay/src/main/java/io/sentry/android/replay/video/SimpleVideoEncoder.kt @@ -34,20 +34,26 @@ import android.graphics.Bitmap import android.media.MediaCodec import android.media.MediaCodecInfo import android.media.MediaFormat -import android.os.Handler -import android.os.Looper import android.view.Surface +import io.sentry.SentryLevel.DEBUG +import io.sentry.SentryOptions +import io.sentry.android.replay.ScreenshotRecorderConfig import java.io.File +import java.nio.ByteBuffer + +private const val TIMEOUT_USEC = 100_000L @TargetApi(26) internal class SimpleVideoEncoder( - val muxerConfig: MuxerConfig + val options: SentryOptions, + val muxerConfig: MuxerConfig, + val onClose: (() -> Unit)? = null ) { private val mediaFormat: MediaFormat = run { val format = MediaFormat.createVideoFormat( muxerConfig.mimeType, - muxerConfig.videoWidth, - muxerConfig.videoHeight + muxerConfig.recorderConfig.recordingWidth, + muxerConfig.recorderConfig.recordingHeight ) // Set some properties. Failing to specify some of these can cause the MediaCodec @@ -63,7 +69,7 @@ internal class SimpleVideoEncoder( format } - private val mediaCodec: MediaCodec = run { + internal val mediaCodec: MediaCodec = run { // val codecs = MediaCodecList(REGULAR_CODECS) // val codecName = codecs.findEncoderForFormat(mediaFormat) // val codec = MediaCodec.createByCodecName(codecName) @@ -72,84 +78,103 @@ internal class SimpleVideoEncoder( codec } + private val bufferInfo: MediaCodec.BufferInfo = MediaCodec.BufferInfo() private val frameMuxer = muxerConfig.frameMuxer + val duration get() = frameMuxer.getVideoTime() private var surface: Surface? = null fun start() { - mediaCodec.setCallback(createMediaCodecCallback(), Handler(Looper.getMainLooper())) - mediaCodec.configure(mediaFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE) surface = mediaCodec.createInputSurface() mediaCodec.start() + drainCodec(false) } - private fun createMediaCodecCallback(): MediaCodec.Callback { - return object : MediaCodec.Callback() { - override fun onInputBufferAvailable(codec: MediaCodec, index: Int) { - } - - override fun onOutputBufferAvailable( - codec: MediaCodec, - index: Int, - info: MediaCodec.BufferInfo - ) { - val encodedData = codec.getOutputBuffer(index)!! + fun encode(image: Bitmap) { + // NOTE do not use `lockCanvas` like what is done in bitmap2video + // This is because https://developer.android.com/reference/android/media/MediaCodec#createInputSurface() + // says that, "Surface.lockCanvas(android.graphics.Rect) may fail or produce unexpected results." + val canvas = surface?.lockHardwareCanvas() + canvas?.drawBitmap(image, 0f, 0f, null) + surface?.unlockCanvasAndPost(canvas) + drainCodec(false) + } - var effectiveSize = info.size + /** + * Extracts all pending data from the encoder. + * + * + * If endOfStream is not set, this returns when there is no more data to drain. If it + * is set, we send EOS to the encoder, and then iterate until we see EOS on the output. + * Calling this with endOfStream set should be done once, right before stopping the muxer. + * + * Borrows heavily from https://bigflake.com/mediacodec/EncodeAndMuxTest.java.txt + */ + private fun drainCodec(endOfStream: Boolean) { + options.logger.log(DEBUG, "[Encoder]: drainCodec($endOfStream)") + if (endOfStream) { + options.logger.log(DEBUG, "[Encoder]: sending EOS to encoder") + mediaCodec.signalEndOfInputStream() + } + var encoderOutputBuffers: Array? = mediaCodec.outputBuffers + while (true) { + val encoderStatus: Int = mediaCodec.dequeueOutputBuffer(bufferInfo, TIMEOUT_USEC) + if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { + // no output available yet + if (!endOfStream) { + break // out of while + } else { + options.logger.log(DEBUG, "[Encoder]: no output available, spinning to await EOS") + } + } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { + // not expected for an encoder + encoderOutputBuffers = mediaCodec.outputBuffers + } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { + // should happen before receiving buffers, and should only happen once + if (frameMuxer.isStarted()) { + throw RuntimeException("format changed twice") + } + val newFormat: MediaFormat = mediaCodec.outputFormat + options.logger.log(DEBUG, "[Encoder]: encoder output format changed: $newFormat") - if (info.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { + // now that we have the Magic Goodies, start the muxer + frameMuxer.start(newFormat) + } else if (encoderStatus < 0) { + options.logger.log(DEBUG, "[Encoder]: unexpected result from encoder.dequeueOutputBuffer: $encoderStatus") + // let's ignore it + } else { + val encodedData = encoderOutputBuffers?.get(encoderStatus) + ?: throw RuntimeException("encoderOutputBuffer $encoderStatus was null") + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_CODEC_CONFIG != 0) { // The codec config data was pulled out and fed to the muxer when we got // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it. - effectiveSize = 0 + options.logger.log(DEBUG, "[Encoder]: ignoring BUFFER_FLAG_CODEC_CONFIG") + bufferInfo.size = 0 } - - if (effectiveSize != 0) { + if (bufferInfo.size != 0) { if (!frameMuxer.isStarted()) { throw RuntimeException("muxer hasn't started") } - frameMuxer.muxVideoFrame(encodedData, info) + frameMuxer.muxVideoFrame(encodedData, bufferInfo) + options.logger.log(DEBUG, "[Encoder]: sent ${bufferInfo.size} bytes to muxer") } - - mediaCodec.releaseOutputBuffer(index, false) - - if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { - actualRelease() - } - } - - override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) { - } - - override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) { - // should happen before receiving buffers, and should only happen once - if (frameMuxer.isStarted()) { - throw RuntimeException("format changed twice") + mediaCodec.releaseOutputBuffer(encoderStatus, false) + if (bufferInfo.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) { + if (!endOfStream) { + options.logger.log(DEBUG, "[Encoder]: reached end of stream unexpectedly") + } else { + options.logger.log(DEBUG, "[Encoder]: end of stream reached") + } + break // out of while } - val newFormat: MediaFormat = mediaCodec.outputFormat - // now that we have the Magic Goodies, start the muxer - frameMuxer.start(newFormat) } } } - fun encode(image: Bitmap) { - // NOTE do not use `lockCanvas` like what is done in bitmap2video - // This is because https://developer.android.com/reference/android/media/MediaCodec#createInputSurface() - // says that, "Surface.lockCanvas(android.graphics.Rect) may fail or produce unexpected results." - val canvas = surface?.lockHardwareCanvas() - canvas?.drawBitmap(image, 0f, 0f, null) - surface?.unlockCanvasAndPost(canvas) - } - - /** - * can only *start* releasing, since it is asynchronous - */ - fun startRelease() { - mediaCodec.signalEndOfInputStream() - } - - private fun actualRelease() { + fun release() { + onClose?.invoke() + drainCodec(true) mediaCodec.stop() mediaCodec.release() surface?.release() @@ -161,11 +186,9 @@ internal class SimpleVideoEncoder( @TargetApi(24) internal data class MuxerConfig( val file: File, - val videoWidth: Int, - val videoHeight: Int, - val scaleFactor: Float, + val recorderConfig: ScreenshotRecorderConfig, + val bitrate: Int = 20_000, + val frameRate: Float = recorderConfig.frameRate.toFloat(), val mimeType: String = MediaFormat.MIMETYPE_VIDEO_AVC, - val frameRate: Float, - val bitrate: Int, val frameMuxer: SimpleFrameMuxer = SimpleMp4FrameMuxer(file.absolutePath, frameRate) ) diff --git a/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt new file mode 100644 index 00000000000..fab5f28ac81 --- /dev/null +++ b/sentry-android-replay/src/test/java/io/sentry/android/replay/ReplayCacheTest.kt @@ -0,0 +1,247 @@ +package io.sentry.android.replay + +import android.graphics.Bitmap +import android.graphics.Bitmap.CompressFormat.JPEG +import android.graphics.Bitmap.Config.ARGB_8888 +import android.media.MediaCodec +import androidx.test.ext.junit.runners.AndroidJUnit4 +import io.sentry.SentryOptions +import io.sentry.android.replay.video.MuxerConfig +import io.sentry.android.replay.video.SimpleVideoEncoder +import io.sentry.protocol.SentryId +import org.junit.Rule +import org.junit.rules.TemporaryFolder +import org.junit.runner.RunWith +import org.robolectric.annotation.Config +import java.io.File +import java.util.concurrent.TimeUnit.MICROSECONDS +import java.util.concurrent.TimeUnit.MILLISECONDS +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertNull +import kotlin.test.assertTrue + +@RunWith(AndroidJUnit4::class) +@Config(sdk = [26]) +class ReplayCacheTest { + + @get:Rule + val tmpDir = TemporaryFolder() + + internal class Fixture { + val options = SentryOptions() + var encoder: SimpleVideoEncoder? = null + fun getSut( + dir: TemporaryFolder?, + replayId: SentryId = SentryId(), + frameRate: Int, + framesToEncode: Int = 0 + ): ReplayCache { + val recorderConfig = ScreenshotRecorderConfig(100, 200, 1f, frameRate) + options.run { + cacheDirPath = dir?.newFolder()?.absolutePath + } + return ReplayCache(options, replayId, recorderConfig, encoderCreator = { videoFile -> + encoder = SimpleVideoEncoder( + options, + MuxerConfig( + file = videoFile, + recorderConfig = recorderConfig, + frameRate = recorderConfig.frameRate.toFloat(), + bitrate = 20 * 1000 + ), + onClose = { + encodeFrame(framesToEncode, frameRate, size = 0, flags = MediaCodec.BUFFER_FLAG_END_OF_STREAM) + } + ).also { it.start() } + repeat(framesToEncode) { encodeFrame(it, frameRate) } + + encoder!! + }) + } + + fun encodeFrame(index: Int, frameRate: Int, size: Int = 10, flags: Int = 0) { + val presentationTime = MICROSECONDS.convert(index * (1000L / frameRate), MILLISECONDS) + encoder!!.mediaCodec.dequeueInputBuffer(0) + encoder!!.mediaCodec.queueInputBuffer(index, index * size, size, presentationTime, flags) + } + } + + private val fixture = Fixture() + + @Test + fun `when no cacheDirPath specified, does not store screenshots`() { + val replayId = SentryId() + val replayCache = fixture.getSut( + null, + replayId, + frameRate = 1 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + + assertTrue(replayCache.frames.isEmpty()) + } + + @Test + fun `stores screenshots with timestamp as name`() { + val replayId = SentryId() + val replayCache = fixture.getSut( + tmpDir, + replayId, + frameRate = 1 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + + val expectedScreenshotFile = File(replayCache.replayCacheDir, "1.jpg") + assertTrue(expectedScreenshotFile.exists()) + assertEquals(replayCache.frames.first().timestamp, 1) + assertEquals(replayCache.frames.first().screenshot, expectedScreenshotFile) + } + + @Test + fun `when no frames are provided, returns nothing`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1 + ) + + val video = replayCache.createVideoOf(5000L, 0, 0) + + assertNull(video) + } + + @Test + fun `deletes frames after creating a video`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1, + framesToEncode = 3 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + replayCache.addFrame(bitmap, 1001) + replayCache.addFrame(bitmap, 2001) + + val segment0 = replayCache.createVideoOf(3000L, 0, 0) + assertEquals(3, segment0!!.frameCount) + assertEquals(3000, segment0.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) + + assertTrue(replayCache.frames.isEmpty()) + assertTrue(replayCache.replayCacheDir!!.listFiles()!!.none { it.extension == "jpg" }) + } + + @Test + fun `repeats last known frame for the segment duration`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1, + framesToEncode = 5 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + + val segment0 = replayCache.createVideoOf(5000L, 0, 0) + assertEquals(5, segment0!!.frameCount) + assertEquals(5000, segment0.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) + } + + @Test + fun `repeats last known frame for the segment duration for each timespan`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1, + framesToEncode = 5 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + replayCache.addFrame(bitmap, 3001) + + val segment0 = replayCache.createVideoOf(5000L, 0, 0) + assertEquals(5, segment0!!.frameCount) + assertEquals(5000, segment0.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) + } + + @Test + fun `repeats last known frame for each segment`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1, + framesToEncode = 5 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + replayCache.addFrame(bitmap, 5001) + + val segment0 = replayCache.createVideoOf(5000L, 0, 0) + assertEquals(5, segment0!!.frameCount) + assertEquals(5000, segment0.duration) + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) + + val segment1 = replayCache.createVideoOf(5000L, 5000L, 1) + assertEquals(5, segment1!!.frameCount) + assertEquals(5000, segment1.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } + assertEquals(File(replayCache.replayCacheDir, "1.mp4"), segment1.video) + } + + @Test + fun `respects frameRate`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 2, + framesToEncode = 6 + ) + + val bitmap = Bitmap.createBitmap(1, 1, ARGB_8888) + replayCache.addFrame(bitmap, 1) + replayCache.addFrame(bitmap, 1001) + replayCache.addFrame(bitmap, 1501) + + val segment0 = replayCache.createVideoOf(3000L, 0, 0) + assertEquals(6, segment0!!.frameCount) + assertEquals(3000, segment0.duration) + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } + assertEquals(File(replayCache.replayCacheDir, "0.mp4"), segment0.video) + } + + @Test + fun `addFrame with File path works`() { + val replayCache = fixture.getSut( + tmpDir, + frameRate = 1, + framesToEncode = 5 + ) + + val flutterCacheDir = + File(fixture.options.cacheDirPath!!, "flutter_replay").also { it.mkdirs() } + val screenshot = File(flutterCacheDir, "1.jpg").also { it.createNewFile() } + val video = File(flutterCacheDir, "flutter_0.mp4") + + screenshot.outputStream().use { + Bitmap.createBitmap(1, 1, ARGB_8888).compress(JPEG, 80, it) + it.flush() + } + replayCache.addFrame(screenshot, frameTimestamp = 1) + + val segment0 = replayCache.createVideoOf(5000L, 0, 0, videoFile = video) + assertEquals(5, segment0!!.frameCount) + assertEquals(5000, segment0.duration) + + assertTrue { segment0.video.exists() && segment0.video.length() > 0 } + assertEquals(File(flutterCacheDir, "flutter_0.mp4"), segment0.video) + } +} diff --git a/sentry-android/build.gradle.kts b/sentry-android/build.gradle.kts index 47b873ac490..81619b736f2 100644 --- a/sentry-android/build.gradle.kts +++ b/sentry-android/build.gradle.kts @@ -35,4 +35,5 @@ android { dependencies { api(projects.sentryAndroidCore) api(projects.sentryAndroidNdk) + api(projects.sentryAndroidReplay) } diff --git a/sentry/api/sentry.api b/sentry/api/sentry.api index b6acae0cdee..3ffc0cda8fc 100644 --- a/sentry/api/sentry.api +++ b/sentry/api/sentry.api @@ -664,6 +664,7 @@ public abstract interface class io/sentry/IScope { public abstract fun getLevel ()Lio/sentry/SentryLevel; public abstract fun getOptions ()Lio/sentry/SentryOptions; public abstract fun getPropagationContext ()Lio/sentry/PropagationContext; + public abstract fun getReplayId ()Lio/sentry/protocol/SentryId; public abstract fun getRequest ()Lio/sentry/protocol/Request; public abstract fun getScreen ()Ljava/lang/String; public abstract fun getSession ()Lio/sentry/Session; @@ -686,6 +687,7 @@ public abstract interface class io/sentry/IScope { public abstract fun setFingerprint (Ljava/util/List;)V public abstract fun setLevel (Lio/sentry/SentryLevel;)V public abstract fun setPropagationContext (Lio/sentry/PropagationContext;)V + public abstract fun setReplayId (Lio/sentry/protocol/SentryId;)V public abstract fun setRequest (Lio/sentry/protocol/Request;)V public abstract fun setScreen (Ljava/lang/String;)V public abstract fun setTag (Ljava/lang/String;Ljava/lang/String;)V @@ -1215,6 +1217,7 @@ public final class io/sentry/NoOpScope : io/sentry/IScope { public fun getLevel ()Lio/sentry/SentryLevel; public fun getOptions ()Lio/sentry/SentryOptions; public fun getPropagationContext ()Lio/sentry/PropagationContext; + public fun getReplayId ()Lio/sentry/protocol/SentryId; public fun getRequest ()Lio/sentry/protocol/Request; public fun getScreen ()Ljava/lang/String; public fun getSession ()Lio/sentry/Session; @@ -1237,6 +1240,7 @@ public final class io/sentry/NoOpScope : io/sentry/IScope { public fun setFingerprint (Ljava/util/List;)V public fun setLevel (Lio/sentry/SentryLevel;)V public fun setPropagationContext (Lio/sentry/PropagationContext;)V + public fun setReplayId (Lio/sentry/protocol/SentryId;)V public fun setRequest (Lio/sentry/protocol/Request;)V public fun setScreen (Ljava/lang/String;)V public fun setTag (Ljava/lang/String;Ljava/lang/String;)V @@ -1639,6 +1643,7 @@ public final class io/sentry/Scope : io/sentry/IScope { public fun getLevel ()Lio/sentry/SentryLevel; public fun getOptions ()Lio/sentry/SentryOptions; public fun getPropagationContext ()Lio/sentry/PropagationContext; + public fun getReplayId ()Lio/sentry/protocol/SentryId; public fun getRequest ()Lio/sentry/protocol/Request; public fun getScreen ()Ljava/lang/String; public fun getSession ()Lio/sentry/Session; @@ -1661,6 +1666,7 @@ public final class io/sentry/Scope : io/sentry/IScope { public fun setFingerprint (Ljava/util/List;)V public fun setLevel (Lio/sentry/SentryLevel;)V public fun setPropagationContext (Lio/sentry/PropagationContext;)V + public fun setReplayId (Lio/sentry/protocol/SentryId;)V public fun setRequest (Lio/sentry/protocol/Request;)V public fun setScreen (Ljava/lang/String;)V public fun setTag (Ljava/lang/String;Ljava/lang/String;)V @@ -4767,7 +4773,7 @@ public final class io/sentry/rrweb/RRWebVideoEvent : io/sentry/rrweb/RRWebEvent, public fun equals (Ljava/lang/Object;)Z public fun getContainer ()Ljava/lang/String; public fun getDataUnknown ()Ljava/util/Map; - public fun getDuration ()I + public fun getDurationMs ()J public fun getEncoding ()Ljava/lang/String; public fun getFrameCount ()I public fun getFrameRate ()I @@ -4785,7 +4791,7 @@ public final class io/sentry/rrweb/RRWebVideoEvent : io/sentry/rrweb/RRWebEvent, public fun serialize (Lio/sentry/ObjectWriter;Lio/sentry/ILogger;)V public fun setContainer (Ljava/lang/String;)V public fun setDataUnknown (Ljava/util/Map;)V - public fun setDuration (I)V + public fun setDurationMs (J)V public fun setEncoding (Ljava/lang/String;)V public fun setFrameCount (I)V public fun setFrameRate (I)V diff --git a/sentry/src/main/java/io/sentry/IScope.java b/sentry/src/main/java/io/sentry/IScope.java index 3842fb2c3a8..4b5930bb544 100644 --- a/sentry/src/main/java/io/sentry/IScope.java +++ b/sentry/src/main/java/io/sentry/IScope.java @@ -2,6 +2,7 @@ import io.sentry.protocol.Contexts; import io.sentry.protocol.Request; +import io.sentry.protocol.SentryId; import io.sentry.protocol.User; import java.util.Collection; import java.util.List; @@ -84,6 +85,23 @@ public interface IScope { @ApiStatus.Internal void setScreen(final @Nullable String screen); + /** + * Returns the Scope's current replay_id, previously set by {@link IScope#setReplayId(SentryId)} + * + * @return the id of the current session replay + */ + @ApiStatus.Internal + @NotNull + SentryId getReplayId(); + + /** + * Sets the Scope's current replay_id + * + * @param replayId the id of the current session replay + */ + @ApiStatus.Internal + void setReplayId(final @NotNull SentryId replayId); + /** * Returns the Scope's request * diff --git a/sentry/src/main/java/io/sentry/NoOpScope.java b/sentry/src/main/java/io/sentry/NoOpScope.java index c756fb49a39..d2be23eba81 100644 --- a/sentry/src/main/java/io/sentry/NoOpScope.java +++ b/sentry/src/main/java/io/sentry/NoOpScope.java @@ -2,6 +2,7 @@ import io.sentry.protocol.Contexts; import io.sentry.protocol.Request; +import io.sentry.protocol.SentryId; import io.sentry.protocol.User; import java.util.ArrayDeque; import java.util.ArrayList; @@ -68,6 +69,14 @@ public void setUser(@Nullable User user) {} @Override public void setScreen(@Nullable String screen) {} + @Override + public @NotNull SentryId getReplayId() { + return SentryId.EMPTY_ID; + } + + @Override + public void setReplayId(@Nullable SentryId replayId) {} + @Override public @Nullable Request getRequest() { return null; diff --git a/sentry/src/main/java/io/sentry/Scope.java b/sentry/src/main/java/io/sentry/Scope.java index 91c9fcd8cfe..161502a9d30 100644 --- a/sentry/src/main/java/io/sentry/Scope.java +++ b/sentry/src/main/java/io/sentry/Scope.java @@ -3,6 +3,7 @@ import io.sentry.protocol.App; import io.sentry.protocol.Contexts; import io.sentry.protocol.Request; +import io.sentry.protocol.SentryId; import io.sentry.protocol.TransactionNameSource; import io.sentry.protocol.User; import io.sentry.util.CollectionUtils; @@ -80,6 +81,9 @@ public final class Scope implements IScope { private @NotNull PropagationContext propagationContext; + /** Scope's session replay id */ + private @NotNull SentryId replayId = SentryId.EMPTY_ID; + /** * Scope's ctor * @@ -101,6 +105,7 @@ private Scope(final @NotNull Scope scope) { final User userRef = scope.user; this.user = userRef != null ? new User(userRef) : null; this.screen = scope.screen; + this.replayId = scope.replayId; final Request requestRef = scope.request; this.request = requestRef != null ? new Request(requestRef) : null; @@ -312,6 +317,18 @@ public void setScreen(final @Nullable String screen) { } } + @Override + public @NotNull SentryId getReplayId() { + return replayId; + } + + @Override + public void setReplayId(final @NotNull SentryId replayId) { + this.replayId = replayId; + + // TODO: set to contexts and notify observers to persist this as well + } + /** * Returns the Scope's request * diff --git a/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java b/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java index ff162f44642..728b28d0e2d 100644 --- a/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java +++ b/sentry/src/main/java/io/sentry/SentryEnvelopeItem.java @@ -23,7 +23,7 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.nio.charset.Charset; -import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.concurrent.Callable; import org.jetbrains.annotations.ApiStatus; @@ -360,7 +360,8 @@ public static SentryEnvelopeItem fromReplay( try (final ByteArrayOutputStream stream = new ByteArrayOutputStream(); final Writer writer = new BufferedWriter(new OutputStreamWriter(stream, UTF_8))) { - final Map replayPayload = new HashMap<>(); + // relay expects the payload to be in this exact order: [event,rrweb,video] + final Map replayPayload = new LinkedHashMap<>(); // first serialize replay event json bytes serializer.serialize(replayEvent, writer); replayPayload.put(SentryItemType.ReplayEvent.getItemType(), stream.toByteArray()); diff --git a/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java b/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java index 5bea9e3c471..1ba9f19c728 100644 --- a/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java +++ b/sentry/src/main/java/io/sentry/rrweb/RRWebVideoEvent.java @@ -26,7 +26,7 @@ public final class RRWebVideoEvent extends RRWebEvent implements JsonUnknown, Js private @NotNull String tag; private int segmentId; private long size; - private int duration; + private long durationMs; private @NotNull String encoding = REPLAY_ENCODING; private @NotNull String container = REPLAY_CONTAINER; private int height; @@ -72,12 +72,12 @@ public void setSize(final long size) { this.size = size; } - public int getDuration() { - return duration; + public long getDurationMs() { + return durationMs; } - public void setDuration(final int duration) { - this.duration = duration; + public void setDurationMs(final long durationMs) { + this.durationMs = durationMs; } @NotNull @@ -189,7 +189,7 @@ public boolean equals(Object o) { RRWebVideoEvent that = (RRWebVideoEvent) o; return segmentId == that.segmentId && size == that.size - && duration == that.duration + && durationMs == that.durationMs && height == that.height && width == that.width && frameCount == that.frameCount @@ -209,7 +209,7 @@ public int hashCode() { tag, segmentId, size, - duration, + durationMs, encoding, container, height, @@ -279,7 +279,7 @@ private void serializePayload(final @NotNull ObjectWriter writer, final @NotNull writer.beginObject(); writer.name(JsonKeys.SEGMENT_ID).value(segmentId); writer.name(JsonKeys.SIZE).value(size); - writer.name(JsonKeys.DURATION).value(duration); + writer.name(JsonKeys.DURATION).value(durationMs); writer.name(JsonKeys.ENCODING).value(encoding); writer.name(JsonKeys.CONTAINER).value(container); writer.name(JsonKeys.HEIGHT).value(height); @@ -380,7 +380,7 @@ private void deserializePayload( event.size = size == null ? 0 : size; break; case JsonKeys.DURATION: - event.duration = reader.nextInt(); + event.durationMs = reader.nextLong(); break; case JsonKeys.CONTAINER: final String container = reader.nextStringOrNull(); diff --git a/sentry/src/test/java/io/sentry/rrweb/RRWebVideoEventSerializationTest.kt b/sentry/src/test/java/io/sentry/rrweb/RRWebVideoEventSerializationTest.kt index 79bfd024564..17a790b5cde 100644 --- a/sentry/src/test/java/io/sentry/rrweb/RRWebVideoEventSerializationTest.kt +++ b/sentry/src/test/java/io/sentry/rrweb/RRWebVideoEventSerializationTest.kt @@ -17,7 +17,7 @@ class RRWebVideoEventSerializationTest { tag = "video" segmentId = 0 size = 4_000_000L - duration = 5000 + durationMs = 5000 height = 1920 width = 1080 frameCount = 5