[SR] Capture Replays for ANRs and crashes (#3565)

Author: romtsn

CHANGELOG.md

@@ -2,6 +2,19 @@
 
 ## Unreleased
 
+### Features
+
+- Session Replay: ([#3565](https://github.com/getsentry/sentry-java/pull/3565)) ([#3609](https://github.com/getsentry/sentry-java/pull/3609))
+  - Capture remaining replay segment for ANRs on next app launch
+  - Capture remaining replay segment for unhandled crashes on next app launch
+
+### Fixes
+
+- Session Replay: ([#3565](https://github.com/getsentry/sentry-java/pull/3565)) ([#3609](https://github.com/getsentry/sentry-java/pull/3609))
+  - Fix stopping replay in `session` mode at 1 hour deadline
+  - Never encode full frames for a video segment, only do partial updates. This further reduces size of the replay segment
+  - Use propagation context when no active transaction for ANRs
+
 ### Dependencies
 
 - Bump Spring Boot to 3.3.2 ([#3541](https://github.com/getsentry/sentry-java/pull/3541))

sentry-android-core/src/main/java/io/sentry/android/core/AnrV2EventProcessor.java

@@ -4,16 +4,19 @@
 import static io.sentry.cache.PersistingOptionsObserver.ENVIRONMENT_FILENAME;
 import static io.sentry.cache.PersistingOptionsObserver.PROGUARD_UUID_FILENAME;
 import static io.sentry.cache.PersistingOptionsObserver.RELEASE_FILENAME;
+import static io.sentry.cache.PersistingOptionsObserver.REPLAY_ERROR_SAMPLE_RATE_FILENAME;
 import static io.sentry.cache.PersistingOptionsObserver.SDK_VERSION_FILENAME;
 import static io.sentry.cache.PersistingScopeObserver.BREADCRUMBS_FILENAME;
 import static io.sentry.cache.PersistingScopeObserver.CONTEXTS_FILENAME;
 import static io.sentry.cache.PersistingScopeObserver.EXTRAS_FILENAME;
 import static io.sentry.cache.PersistingScopeObserver.FINGERPRINT_FILENAME;
 import static io.sentry.cache.PersistingScopeObserver.LEVEL_FILENAME;
+import static io.sentry.cache.PersistingScopeObserver.REPLAY_FILENAME;
 import static io.sentry.cache.PersistingScopeObserver.REQUEST_FILENAME;
 import static io.sentry.cache.PersistingScopeObserver.TRACE_FILENAME;
 import static io.sentry.cache.PersistingScopeObserver.TRANSACTION_FILENAME;
 import static io.sentry.cache.PersistingScopeObserver.USER_FILENAME;
+import static io.sentry.protocol.Contexts.REPLAY_ID;
 
 import android.annotation.SuppressLint;
 import android.app.ActivityManager;
@@ -51,6 +54,8 @@
 import io.sentry.protocol.SentryTransaction;
 import io.sentry.protocol.User;
 import io.sentry.util.HintUtils;
+import java.io.File;
+import java.security.SecureRandom;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.Collections;
@@ -78,13 +83,24 @@ public final class AnrV2EventProcessor implements BackfillingEventProcessor {
 
   private final @NotNull SentryExceptionFactory sentryExceptionFactory;
 
+  private final @Nullable SecureRandom random;
+
   public AnrV2EventProcessor(
       final @NotNull Context context,
       final @NotNull SentryAndroidOptions options,
       final @NotNull BuildInfoProvider buildInfoProvider) {
+    this(context, options, buildInfoProvider, null);
+  }
+
+  AnrV2EventProcessor(
+      final @NotNull Context context,
+      final @NotNull SentryAndroidOptions options,
+      final @NotNull BuildInfoProvider buildInfoProvider,
+      final @Nullable SecureRandom random) {
     this.context = context;
     this.options = options;
     this.buildInfoProvider = buildInfoProvider;
+    this.random = random;
 
     final SentryStackTraceFactory sentryStackTraceFactory =
         new SentryStackTraceFactory(this.options);
@@ -151,6 +167,72 @@ private void backfillScope(final @NotNull SentryEvent event, final @NotNull Obje
     setFingerprints(event, hint);
     setLevel(event);
     setTrace(event);
+    setReplayId(event);
+  }
+
+  private boolean sampleReplay(final @NotNull SentryEvent event) {
+    final @Nullable String replayErrorSampleRate =
+        PersistingOptionsObserver.read(options, REPLAY_ERROR_SAMPLE_RATE_FILENAME, String.class);
+
+    if (replayErrorSampleRate == null) {
+      return false;
+    }
+
+    try {
+      // we have to sample here with the old sample rate, because it may change between app launches
+      final @NotNull SecureRandom random = this.random != null ? this.random : new SecureRandom();
+      final double replayErrorSampleRateDouble = Double.parseDouble(replayErrorSampleRate);
+      if (replayErrorSampleRateDouble < random.nextDouble()) {
+        options
+            .getLogger()
+            .log(
+                SentryLevel.DEBUG,
+                "Not capturing replay for ANR %s due to not being sampled.",
+                event.getEventId());
+        return false;
+      }
+    } catch (Throwable e) {
+      options.getLogger().log(SentryLevel.ERROR, "Error parsing replay sample rate.", e);
+      return false;
+    }
+
+    return true;
+  }
+
+  private void setReplayId(final @NotNull SentryEvent event) {
+    @Nullable
+    String persistedReplayId = PersistingScopeObserver.read(options, REPLAY_FILENAME, String.class);
+    final @NotNull File replayFolder =
+        new File(options.getCacheDirPath(), "replay_" + persistedReplayId);
+    if (!replayFolder.exists()) {
+      if (!sampleReplay(event)) {
+        return;
+      }
+      // if the replay folder does not exist (e.g. running in buffer mode), we need to find the
+      // latest replay folder that was modified before the ANR event.
+      persistedReplayId = null;
+      long lastModified = Long.MIN_VALUE;
+      final File[] dirs = new File(options.getCacheDirPath()).listFiles();
+      if (dirs != null) {
+        for (File dir : dirs) {
+          if (dir.isDirectory() && dir.getName().startsWith("replay_")) {
+            if (dir.lastModified() > lastModified
+                && dir.lastModified() <= event.getTimestamp().getTime()) {
+              lastModified = dir.lastModified();
+              persistedReplayId = dir.getName().substring("replay_".length());
+            }
+          }
+        }
+      }
+    }
+
+    if (persistedReplayId == null) {
+      return;
+    }
+
+    // store the relevant replayId so ReplayIntegration can pick it up and finalize that replay
+    PersistingScopeObserver.store(options, persistedReplayId, REPLAY_FILENAME);
+    event.getContexts().put(REPLAY_ID, persistedReplayId);
   }
 
   private void setTrace(final @NotNull SentryEvent event) {

sentry-android-core/src/test/java/io/sentry/android/core/AnrV2EventProcessorTest.kt

@@ -15,18 +15,20 @@ import io.sentry.SentryEvent
 import io.sentry.SentryLevel
 import io.sentry.SentryLevel.DEBUG
 import io.sentry.SpanContext
-import io.sentry.cache.PersistingOptionsObserver
 import io.sentry.cache.PersistingOptionsObserver.DIST_FILENAME
 import io.sentry.cache.PersistingOptionsObserver.ENVIRONMENT_FILENAME
 import io.sentry.cache.PersistingOptionsObserver.OPTIONS_CACHE
 import io.sentry.cache.PersistingOptionsObserver.PROGUARD_UUID_FILENAME
 import io.sentry.cache.PersistingOptionsObserver.RELEASE_FILENAME
+import io.sentry.cache.PersistingOptionsObserver.REPLAY_ERROR_SAMPLE_RATE_FILENAME
 import io.sentry.cache.PersistingOptionsObserver.SDK_VERSION_FILENAME
+import io.sentry.cache.PersistingScopeObserver
 import io.sentry.cache.PersistingScopeObserver.BREADCRUMBS_FILENAME
 import io.sentry.cache.PersistingScopeObserver.CONTEXTS_FILENAME
 import io.sentry.cache.PersistingScopeObserver.EXTRAS_FILENAME
 import io.sentry.cache.PersistingScopeObserver.FINGERPRINT_FILENAME
 import io.sentry.cache.PersistingScopeObserver.LEVEL_FILENAME
+import io.sentry.cache.PersistingScopeObserver.REPLAY_FILENAME
 import io.sentry.cache.PersistingScopeObserver.REQUEST_FILENAME
 import io.sentry.cache.PersistingScopeObserver.SCOPE_CACHE
 import io.sentry.cache.PersistingScopeObserver.TAGS_FILENAME
@@ -44,6 +46,7 @@ import io.sentry.protocol.OperatingSystem
 import io.sentry.protocol.Request
 import io.sentry.protocol.Response
 import io.sentry.protocol.SdkVersion
+import io.sentry.protocol.SentryId
 import io.sentry.protocol.SentryStackFrame
 import io.sentry.protocol.SentryStackTrace
 import io.sentry.protocol.SentryThread
@@ -75,7 +78,9 @@ class AnrV2EventProcessorTest {
     val tmpDir = TemporaryFolder()
 
     class Fixture {
-
+        companion object {
+            const val REPLAY_ID = "64cf554cc8d74c6eafa3e08b7c984f6d"
+        }
         val buildInfo = mock<BuildInfoProvider>()
         lateinit var context: Context
         val options = SentryAndroidOptions().apply {
@@ -87,7 +92,8 @@ class AnrV2EventProcessorTest {
             dir: TemporaryFolder,
             currentSdk: Int = Build.VERSION_CODES.LOLLIPOP,
             populateScopeCache: Boolean = false,
-            populateOptionsCache: Boolean = false
+            populateOptionsCache: Boolean = false,
+            replayErrorSampleRate: Double? = null
         ): AnrV2EventProcessor {
             options.cacheDirPath = dir.newFolder().absolutePath
             options.environment = "release"
@@ -118,6 +124,7 @@ class AnrV2EventProcessorTest {
                     REQUEST_FILENAME,
                     Request().apply { url = "google.com"; method = "GET" }
                 )
+                persistScope(REPLAY_FILENAME, SentryId(REPLAY_ID))
             }
 
             if (populateOptionsCache) {
@@ -126,7 +133,10 @@ class AnrV2EventProcessorTest {
                 persistOptions(SDK_VERSION_FILENAME, SdkVersion("sentry.java.android", "6.15.0"))
                 persistOptions(DIST_FILENAME, "232")
                 persistOptions(ENVIRONMENT_FILENAME, "debug")
-                persistOptions(PersistingOptionsObserver.TAGS_FILENAME, mapOf("option" to "tag"))
+                persistOptions(TAGS_FILENAME, mapOf("option" to "tag"))
+                replayErrorSampleRate?.let {
+                    persistOptions(REPLAY_ERROR_SAMPLE_RATE_FILENAME, it.toString())
+                }
             }
 
             return AnrV2EventProcessor(context, options, buildInfo)
@@ -544,6 +554,65 @@ class AnrV2EventProcessorTest {
         assertEquals(listOf("{{ default }}", "foreground-anr"), processedForeground.fingerprints)
     }
 
+    @Test
+    fun `sets replayId when replay folder exists`() {
+        val hint = HintUtils.createWithTypeCheckHint(BackfillableHint())
+        val processor = fixture.getSut(tmpDir, populateScopeCache = true)
+        val replayFolder = File(fixture.options.cacheDirPath, "replay_${Fixture.REPLAY_ID}").also { it.mkdirs() }
+
+        val processed = processor.process(SentryEvent(), hint)!!
+
+        assertEquals(Fixture.REPLAY_ID, processed.contexts[Contexts.REPLAY_ID].toString())
+    }
+
+    @Test
+    fun `does not set replayId when replay folder does not exist and no sample rate persisted`() {
+        val hint = HintUtils.createWithTypeCheckHint(BackfillableHint())
+        val processor = fixture.getSut(tmpDir, populateScopeCache = true)
+        val replayId1 = SentryId()
+        val replayId2 = SentryId()
+
+        val replayFolder1 = File(fixture.options.cacheDirPath, "replay_$replayId1").also { it.mkdirs() }
+        val replayFolder2 = File(fixture.options.cacheDirPath, "replay_$replayId2").also { it.mkdirs() }
+
+        val processed = processor.process(SentryEvent(), hint)!!
+
+        assertNull(processed.contexts[Contexts.REPLAY_ID])
+    }
+
+    @Test
+    fun `does not set replayId when replay folder does not exist and not sampled`() {
+        val hint = HintUtils.createWithTypeCheckHint(BackfillableHint())
+        val processor = fixture.getSut(tmpDir, populateScopeCache = true, populateOptionsCache = true, replayErrorSampleRate = 0.0)
+        val replayId1 = SentryId()
+        val replayId2 = SentryId()
+
+        val replayFolder1 = File(fixture.options.cacheDirPath, "replay_$replayId1").also { it.mkdirs() }
+        val replayFolder2 = File(fixture.options.cacheDirPath, "replay_$replayId2").also { it.mkdirs() }
+
+        val processed = processor.process(SentryEvent(), hint)!!
+
+        assertNull(processed.contexts[Contexts.REPLAY_ID])
+    }
+
+    @Test
+    fun `set replayId of the last modified folder`() {
+        val hint = HintUtils.createWithTypeCheckHint(BackfillableHint())
+        val processor = fixture.getSut(tmpDir, populateScopeCache = true, populateOptionsCache = true, replayErrorSampleRate = 1.0)
+        val replayId1 = SentryId()
+        val replayId2 = SentryId()
+
+        val replayFolder1 = File(fixture.options.cacheDirPath, "replay_$replayId1").also { it.mkdirs() }
+        val replayFolder2 = File(fixture.options.cacheDirPath, "replay_$replayId2").also { it.mkdirs() }
+        replayFolder1.setLastModified(1000)
+        replayFolder2.setLastModified(500)
+
+        val processed = processor.process(SentryEvent(), hint)!!
+
+        assertEquals(replayId1.toString(), processed.contexts[Contexts.REPLAY_ID].toString())
+        assertEquals(replayId1.toString(), PersistingScopeObserver.read(fixture.options, REPLAY_FILENAME, String::class.java))
+    }
+
     private fun processEvent(
         hint: Hint,
         populateScopeCache: Boolean = false,

sentry-android-replay/api/sentry-android-replay.api

@@ -34,18 +34,25 @@ public abstract interface class io/sentry/android/replay/Recorder : java/io/Clos
 }
 
 public final class io/sentry/android/replay/ReplayCache : java/io/Closeable {
+	public static final field Companion Lio/sentry/android/replay/ReplayCache$Companion;
 	public fun <init> (Lio/sentry/SentryOptions;Lio/sentry/protocol/SentryId;Lio/sentry/android/replay/ScreenshotRecorderConfig;)V
 	public final fun addFrame (Ljava/io/File;J)V
 	public fun close ()V
 	public final fun createVideoOf (JJIIILjava/io/File;)Lio/sentry/android/replay/GeneratedVideo;
 	public static synthetic fun createVideoOf$default (Lio/sentry/android/replay/ReplayCache;JJIIILjava/io/File;ILjava/lang/Object;)Lio/sentry/android/replay/GeneratedVideo;
+	public final fun persistSegmentValues (Ljava/lang/String;Ljava/lang/String;)V
 	public final fun rotate (J)V
 }
 
+public final class io/sentry/android/replay/ReplayCache$Companion {
+	public final fun makeReplayCacheDir (Lio/sentry/SentryOptions;Lio/sentry/protocol/SentryId;)Ljava/io/File;
+}
+
 public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/TouchRecorderCallback, java/io/Closeable {
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function2;)V
 	public synthetic fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function2;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
+	public fun captureReplay (Ljava/lang/Boolean;)V
 	public fun close ()V
 	public fun getBreadcrumbConverter ()Lio/sentry/ReplayBreadcrumbConverter;
 	public final fun getReplayCacheDir ()Ljava/io/File;
@@ -59,8 +66,6 @@ public final class io/sentry/android/replay/ReplayIntegration : android/content/
 	public fun pause ()V
 	public fun register (Lio/sentry/IHub;Lio/sentry/SentryOptions;)V
 	public fun resume ()V
-	public fun sendReplay (Ljava/lang/Boolean;Ljava/lang/String;Lio/sentry/Hint;)V
-	public fun sendReplayForEvent (Lio/sentry/SentryEvent;Lio/sentry/Hint;)V
 	public fun setBreadcrumbConverter (Lio/sentry/ReplayBreadcrumbConverter;)V
 	public fun start ()V
 	public fun stop ()V

sentry-android-replay/src/main/java/io/sentry/android/replay/DefaultReplayBreadcrumbConverter.kt

@@ -131,14 +131,23 @@ public open class DefaultReplayBreadcrumbConverter : ReplayBreadcrumbConverter {
 
     private fun Breadcrumb.toRRWebSpanEvent(): RRWebSpanEvent {
         val breadcrumb = this
+        val httpStartTimestamp = breadcrumb.data[SpanDataConvention.HTTP_START_TIMESTAMP]
+        val httpEndTimestamp = breadcrumb.data[SpanDataConvention.HTTP_END_TIMESTAMP]
         return RRWebSpanEvent().apply {
             timestamp = breadcrumb.timestamp.time
             op = "resource.http"
             description = breadcrumb.data["url"] as String
-            startTimestamp =
-                (breadcrumb.data[SpanDataConvention.HTTP_START_TIMESTAMP] as Long) / 1000.0
-            endTimestamp =
-                (breadcrumb.data[SpanDataConvention.HTTP_END_TIMESTAMP] as Long) / 1000.0
+            // can be double if it was serialized to disk
+            startTimestamp = if (httpStartTimestamp is Double) {
+                httpStartTimestamp / 1000.0
+            } else {
+                (httpStartTimestamp as Long) / 1000.0
+            }
+            endTimestamp = if (httpEndTimestamp is Double) {
+                httpEndTimestamp / 1000.0
+            } else {
+                (httpEndTimestamp as Long) / 1000.0
+            }
 
             val breadcrumbData = mutableMapOf<String, Any?>()
             for ((key, value) in breadcrumb.data) {