[SR] Gesture/touch support for Flutter (#3623)

* Persist buffer replay type when switching to session

* Ensure no gaps in segment timestamps when converting strategies

* Properly store screen name at start for buffer mode

* Changelog

* Decouple gesture tracking from WindowRecorder

* Format code

* Add tests

* Changelog

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## Unreleased
 
+### Features
+
+- Session Replay: Gesture/touch support for Flutter ([#3623](https://github.com/getsentry/sentry-java/pull/3623))
+
 ### Fixes
 
 - Avoid ArrayIndexOutOfBoundsException on Android cpu data collection ([#3598](https://github.com/getsentry/sentry-java/pull/3598))

sentry-android-replay/api/sentry-android-replay.api

@@ -49,7 +49,7 @@ public final class io/sentry/android/replay/ReplayCache$Companion {
 	public final fun makeReplayCacheDir (Lio/sentry/SentryOptions;Lio/sentry/protocol/SentryId;)Ljava/io/File;
 }
 
-public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/TouchRecorderCallback, java/io/Closeable {
+public final class io/sentry/android/replay/ReplayIntegration : android/content/ComponentCallbacks, io/sentry/Integration, io/sentry/ReplayController, io/sentry/android/replay/ScreenshotRecorderCallback, io/sentry/android/replay/gestures/TouchRecorderCallback, java/io/Closeable {
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;)V
 	public fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function2;)V
 	public synthetic fun <init> (Landroid/content/Context;Lio/sentry/transport/ICurrentDateProvider;Lkotlin/jvm/functions/Function0;Lkotlin/jvm/functions/Function1;Lkotlin/jvm/functions/Function2;ILkotlin/jvm/internal/DefaultConstructorMarker;)V
@@ -103,7 +103,18 @@ public final class io/sentry/android/replay/ScreenshotRecorderConfig$Companion {
 	public final fun from (Landroid/content/Context;Lio/sentry/SentryReplayOptions;)Lio/sentry/android/replay/ScreenshotRecorderConfig;
 }
 
-public abstract interface class io/sentry/android/replay/TouchRecorderCallback {
+public final class io/sentry/android/replay/gestures/GestureRecorder : io/sentry/android/replay/OnRootViewsChangedListener {
+	public fun <init> (Lio/sentry/SentryOptions;Lio/sentry/android/replay/gestures/TouchRecorderCallback;)V
+	public fun onRootViewsChanged (Landroid/view/View;Z)V
+	public final fun stop ()V
+}
+
+public final class io/sentry/android/replay/gestures/ReplayGestureConverter {
+	public fun <init> (Lio/sentry/transport/ICurrentDateProvider;)V
+	public final fun convert (Landroid/view/MotionEvent;Lio/sentry/android/replay/ScreenshotRecorderConfig;)Ljava/util/List;
+}
+
+public abstract interface class io/sentry/android/replay/gestures/TouchRecorderCallback {
 	public abstract fun onTouchEvent (Landroid/view/MotionEvent;)V
 }
 

sentry-android-replay/src/main/java/io/sentry/android/replay/ReplayIntegration.kt

@@ -20,6 +20,8 @@ import io.sentry.android.replay.capture.BufferCaptureStrategy
 import io.sentry.android.replay.capture.CaptureStrategy
 import io.sentry.android.replay.capture.CaptureStrategy.ReplaySegment
 import io.sentry.android.replay.capture.SessionCaptureStrategy
+import io.sentry.android.replay.gestures.GestureRecorder
+import io.sentry.android.replay.gestures.TouchRecorderCallback
 import io.sentry.android.replay.util.MainLooperHandler
 import io.sentry.android.replay.util.sample
 import io.sentry.android.replay.util.submitSafely
@@ -37,6 +39,7 @@ import java.io.File
 import java.security.SecureRandom
 import java.util.LinkedList
 import java.util.concurrent.atomic.AtomicBoolean
+import kotlin.LazyThreadSafetyMode.NONE
 
 public class ReplayIntegration(
     private val context: Context,
@@ -62,16 +65,20 @@ public class ReplayIntegration(
         recorderConfigProvider: ((configChanged: Boolean) -> ScreenshotRecorderConfig)?,
         replayCacheProvider: ((replayId: SentryId, recorderConfig: ScreenshotRecorderConfig) -> ReplayCache)?,
         replayCaptureStrategyProvider: ((isFullSession: Boolean) -> CaptureStrategy)? = null,
-        mainLooperHandler: MainLooperHandler? = null
+        mainLooperHandler: MainLooperHandler? = null,
+        gestureRecorderProvider: (() -> GestureRecorder)? = null
     ) : this(context, dateProvider, recorderProvider, recorderConfigProvider, replayCacheProvider) {
         this.replayCaptureStrategyProvider = replayCaptureStrategyProvider
         this.mainLooperHandler = mainLooperHandler ?: MainLooperHandler()
+        this.gestureRecorderProvider = gestureRecorderProvider
     }
 
     private lateinit var options: SentryOptions
     private var hub: IHub? = null
     private var recorder: Recorder? = null
+    private var gestureRecorder: GestureRecorder? = null
     private val random by lazy { SecureRandom() }
+    private val rootViewsSpy by lazy(NONE) { RootViewsSpy.install() }
 
     // TODO: probably not everything has to be thread-safe here
     internal val isEnabled = AtomicBoolean(false)
@@ -81,6 +88,7 @@ public class ReplayIntegration(
     private var replayBreadcrumbConverter: ReplayBreadcrumbConverter = NoOpReplayBreadcrumbConverter.getInstance()
     private var replayCaptureStrategyProvider: ((isFullSession: Boolean) -> CaptureStrategy)? = null
     private var mainLooperHandler: MainLooperHandler = MainLooperHandler()
+    private var gestureRecorderProvider: (() -> GestureRecorder)? = null
 
     private lateinit var recorderConfig: ScreenshotRecorderConfig
 
@@ -100,7 +108,8 @@ public class ReplayIntegration(
         }
 
         this.hub = hub
-        recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, this, mainLooperHandler)
+        recorder = recorderProvider?.invoke() ?: WindowRecorder(options, this, mainLooperHandler)
+        gestureRecorder = gestureRecorderProvider?.invoke() ?: GestureRecorder(options, this)
         isEnabled.set(true)
 
         try {
@@ -147,6 +156,7 @@ public class ReplayIntegration(
 
         captureStrategy?.start(recorderConfig)
         recorder?.start(recorderConfig)
+        registerRootViewListeners()
     }
 
     override fun resume() {
@@ -197,7 +207,9 @@ public class ReplayIntegration(
             return
         }
 
+        unregisterRootViewListeners()
         recorder?.stop()
+        gestureRecorder?.stop()
         captureStrategy?.stop()
         isRecording.set(false)
         captureStrategy?.close()
@@ -252,6 +264,20 @@ public class ReplayIntegration(
         captureStrategy?.onTouchEvent(event)
     }
 
+    private fun registerRootViewListeners() {
+        if (recorder is OnRootViewsChangedListener) {
+            rootViewsSpy.listeners += (recorder as OnRootViewsChangedListener)
+        }
+        rootViewsSpy.listeners += gestureRecorder
+    }
+
+    private fun unregisterRootViewListeners() {
+        if (recorder is OnRootViewsChangedListener) {
+            rootViewsSpy.listeners -= (recorder as OnRootViewsChangedListener)
+        }
+        rootViewsSpy.listeners -= gestureRecorder
+    }
+
     private fun cleanupReplays(unfinishedReplayId: String = "") {
         // clean up old replays
         options.cacheDirPath?.let { cacheDir ->

sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt

@@ -288,6 +288,18 @@ public data class ScreenshotRecorderConfig(
     val frameRate: Int,
     val bitRate: Int
 ) {
+    internal constructor(
+        scaleFactorX: Float,
+        scaleFactorY: Float
+    ) : this(
+        recordingWidth = 0,
+        recordingHeight = 0,
+        scaleFactorX = scaleFactorX,
+        scaleFactorY = scaleFactorY,
+        frameRate = 0,
+        bitRate = 0
+    )
+
     companion object {
         /**
          * Since codec block size is 16, so we have to adjust the width and height to it, otherwise

sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt

@@ -1,13 +1,8 @@
 package io.sentry.android.replay
 
 import android.annotation.TargetApi
-import android.view.MotionEvent
 import android.view.View
-import android.view.Window
-import io.sentry.SentryLevel.DEBUG
-import io.sentry.SentryLevel.ERROR
 import io.sentry.SentryOptions
-import io.sentry.android.replay.util.FixedWindowCallback
 import io.sentry.android.replay.util.MainLooperHandler
 import io.sentry.android.replay.util.gracefullyShutdown
 import io.sentry.android.replay.util.scheduleAtFixedRateSafely
@@ -17,24 +12,18 @@ import java.util.concurrent.ScheduledFuture
 import java.util.concurrent.ThreadFactory
 import java.util.concurrent.TimeUnit.MILLISECONDS
 import java.util.concurrent.atomic.AtomicBoolean
-import kotlin.LazyThreadSafetyMode.NONE
 
 @TargetApi(26)
 internal class WindowRecorder(
     private val options: SentryOptions,
     private val screenshotRecorderCallback: ScreenshotRecorderCallback? = null,
-    private val touchRecorderCallback: TouchRecorderCallback? = null,
     private val mainLooperHandler: MainLooperHandler
-) : Recorder {
+) : Recorder, OnRootViewsChangedListener {
 
     internal companion object {
         private const val TAG = "WindowRecorder"
     }
 
-    private val rootViewsSpy by lazy(NONE) {
-        RootViewsSpy.install()
-    }
-
     private val isRecording = AtomicBoolean(false)
     private val rootViews = ArrayList<WeakReference<View>>()
     private var recorder: ScreenshotRecorder? = null
@@ -43,15 +32,11 @@ internal class WindowRecorder(
         Executors.newSingleThreadScheduledExecutor(RecorderExecutorServiceThreadFactory())
     }
 
-    private val onRootViewsChangedListener = OnRootViewsChangedListener { root, added ->
+    override fun onRootViewsChanged(root: View, added: Boolean) {
         if (added) {
             rootViews.add(WeakReference(root))
             recorder?.bind(root)
-
-            root.startGestureTracking()
         } else {
-            root.stopGestureTracking()
-
             recorder?.unbind(root)
             rootViews.removeAll { it.get() == root }
 
@@ -68,11 +53,10 @@ internal class WindowRecorder(
         }
 
         recorder = ScreenshotRecorder(recorderConfig, options, mainLooperHandler, screenshotRecorderCallback)
-        rootViewsSpy.listeners += onRootViewsChangedListener
         capturingTask = capturer.scheduleAtFixedRateSafely(
             options,
             "$TAG.capture",
-            0L,
+            100L, // delay the first run by a bit, to allow root view listener to register
             1000L / recorderConfig.frameRate,
             MILLISECONDS
         ) {
@@ -88,7 +72,6 @@ internal class WindowRecorder(
     }
 
     override fun stop() {
-        rootViewsSpy.listeners -= onRootViewsChangedListener
         rootViews.forEach { recorder?.unbind(it.get()) }
         recorder?.close()
         rootViews.clear()
@@ -103,55 +86,6 @@ internal class WindowRecorder(
         capturer.gracefullyShutdown(options)
     }
 
-    private fun View.startGestureTracking() {
-        val window = phoneWindow
-        if (window == null) {
-            options.logger.log(DEBUG, "Window is invalid, not tracking gestures")
-            return
-        }
-
-        if (touchRecorderCallback == null) {
-            options.logger.log(DEBUG, "TouchRecorderCallback is null, not tracking gestures")
-            return
-        }
-
-        val delegate = window.callback
-        window.callback = SentryReplayGestureRecorder(options, touchRecorderCallback, delegate)
-    }
-
-    private fun View.stopGestureTracking() {
-        val window = phoneWindow
-        if (window == null) {
-            options.logger.log(DEBUG, "Window was null in stopGestureTracking")
-            return
-        }
-
-        if (window.callback is SentryReplayGestureRecorder) {
-            val delegate = (window.callback as SentryReplayGestureRecorder).delegate
-            window.callback = delegate
-        }
-    }
-
-    private class SentryReplayGestureRecorder(
-        private val options: SentryOptions,
-        private val touchRecorderCallback: TouchRecorderCallback?,
-        delegate: Window.Callback?
-    ) : FixedWindowCallback(delegate) {
-        override fun dispatchTouchEvent(event: MotionEvent?): Boolean {
-            if (event != null) {
-                val copy: MotionEvent = MotionEvent.obtainNoHistory(event)
-                try {
-                    touchRecorderCallback?.onTouchEvent(copy)
-                } catch (e: Throwable) {
-                    options.logger.log(ERROR, "Error dispatching touch event", e)
-                } finally {
-                    copy.recycle()
-                }
-            }
-            return super.dispatchTouchEvent(event)
-        }
-    }
-
     private class RecorderExecutorServiceThreadFactory : ThreadFactory {
         private var cnt = 0
         override fun newThread(r: Runnable): Thread {
@@ -161,7 +95,3 @@ internal class WindowRecorder(
         }
     }
 }
-
-public interface TouchRecorderCallback {
-    fun onTouchEvent(event: MotionEvent)
-}