Add Canvas Capture Strategy

CHANGELOG.md

@@ -2,6 +2,17 @@
 
 ## Unreleased
 
+### Features
+
+- Session Replay: Add new experimental Canvas Capture Strategy ([#4777](https://github.com/getsentry/sentry-java/pull/4777))
+  - A new screenshot capture strategy that uses Android's Canvas API for more accurate text masking
+  - Any `.drawText()` calls are replaced with rectangles to ensure no text is not captured
+  ```kotlin
+  SentryAndroid.init(context) { options ->
+    options.sessionReplay.screenshotStrategy = ScreenshotStrategyType.CANVAS
+  }
+  ```
+
 ### Fixes
 
 - Use logger from options for JVM profiler ([#4771](https://github.com/getsentry/sentry-java/pull/4771))

sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt

@@ -4,60 +4,51 @@ import android.annotation.SuppressLint
 import android.annotation.TargetApi
 import android.content.Context
 import android.graphics.Bitmap
-import android.graphics.Canvas
-import android.graphics.Color
-import android.graphics.Matrix
-import android.graphics.Paint
-import android.graphics.Rect
-import android.graphics.RectF
-import android.view.PixelCopy
 import android.view.View
 import android.view.ViewTreeObserver
+import io.sentry.ScreenshotStrategyType
 import io.sentry.SentryLevel.DEBUG
-import io.sentry.SentryLevel.INFO
 import io.sentry.SentryLevel.WARNING
 import io.sentry.SentryOptions
 import io.sentry.SentryReplayOptions
+import io.sentry.android.replay.screenshot.CanvasStrategy
+import io.sentry.android.replay.screenshot.PixelCopyStrategy
+import io.sentry.android.replay.screenshot.ScreenshotStrategy
 import io.sentry.android.replay.util.DebugOverlayDrawable
-import io.sentry.android.replay.util.MainLooperHandler
 import io.sentry.android.replay.util.addOnDrawListenerSafe
-import io.sentry.android.replay.util.getVisibleRects
 import io.sentry.android.replay.util.removeOnDrawListenerSafe
-import io.sentry.android.replay.util.submitSafely
-import io.sentry.android.replay.util.traverse
-import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode
-import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode.ImageViewHierarchyNode
-import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode.TextViewHierarchyNode
 import java.io.File
 import java.lang.ref.WeakReference
-import java.util.concurrent.ScheduledExecutorService
 import java.util.concurrent.atomic.AtomicBoolean
-import kotlin.LazyThreadSafetyMode.NONE
 import kotlin.math.roundToInt
 
 @SuppressLint("UseKtx")
 @TargetApi(26)
 internal class ScreenshotRecorder(
   val config: ScreenshotRecorderConfig,
   val options: SentryOptions,
-  private val mainLooperHandler: MainLooperHandler,
-  private val recorder: ScheduledExecutorService,
-  private val screenshotRecorderCallback: ScreenshotRecorderCallback?,
+  val executorProvider: ExecutorProvider,
+  screenshotRecorderCallback: ScreenshotRecorderCallback?,
 ) : ViewTreeObserver.OnDrawListener {
   private var rootView: WeakReference<View>? = null
-  private val maskingPaint by lazy(NONE) { Paint() }
-  private val singlePixelBitmap: Bitmap by
-    lazy(NONE) { Bitmap.createBitmap(1, 1, Bitmap.Config.ARGB_8888) }
-  private val screenshot =
-    Bitmap.createBitmap(config.recordingWidth, config.recordingHeight, Bitmap.Config.ARGB_8888)
-  private val singlePixelBitmapCanvas: Canvas by lazy(NONE) { Canvas(singlePixelBitmap) }
-  private val prescaledMatrix by
-    lazy(NONE) { Matrix().apply { preScale(config.scaleFactorX, config.scaleFactorY) } }
-  private val contentChanged = AtomicBoolean(false)
   private val isCapturing = AtomicBoolean(true)
-  private val lastCaptureSuccessful = AtomicBoolean(false)
 
   private val debugOverlayDrawable = DebugOverlayDrawable()
+  private val contentChanged = AtomicBoolean(false)
+
+  private val screenshotStrategy: ScreenshotStrategy =
+    when (options.sessionReplay.screenshotStrategy) {
+      ScreenshotStrategyType.CANVAS ->
+        CanvasStrategy(executorProvider, screenshotRecorderCallback, options, config)
+      ScreenshotStrategyType.PIXEL_COPY ->
+        PixelCopyStrategy(
+          executorProvider,
+          screenshotRecorderCallback,
+          options,
+          config,
+          debugOverlayDrawable,
+        )
+    }
 
   fun capture() {
     if (options.sessionReplay.isDebug) {
@@ -75,12 +66,12 @@ internal class ScreenshotRecorder(
         DEBUG,
         "Capturing screenshot, contentChanged: %s, lastCaptureSuccessful: %s",
         contentChanged.get(),
-        lastCaptureSuccessful.get(),
+        screenshotStrategy.lastCaptureSuccessful(),
       )
     }
 
-    if (!contentChanged.get() && lastCaptureSuccessful.get()) {
-      screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
+    if (!contentChanged.get()) {
+      screenshotStrategy.emitLastScreenshot()
       return
     }
 
@@ -98,93 +89,9 @@ internal class ScreenshotRecorder(
 
     try {
       contentChanged.set(false)
-      PixelCopy.request(
-        window,
-        screenshot,
-        { copyResult: Int ->
-          if (copyResult != PixelCopy.SUCCESS) {
-            options.logger.log(INFO, "Failed to capture replay recording: %d", copyResult)
-            lastCaptureSuccessful.set(false)
-            return@request
-          }
-
-          // TODO: handle animations with heuristics (e.g. if we fall under this condition 2 times
-          // in a row, we should capture)
-          if (contentChanged.get()) {
-            options.logger.log(INFO, "Failed to determine view hierarchy, not capturing")
-            lastCaptureSuccessful.set(false)
-            return@request
-          }
-
-          // TODO: disableAllMasking here and dont traverse?
-          val viewHierarchy = ViewHierarchyNode.fromView(root, null, 0, options)
-          root.traverse(viewHierarchy, options)
-
-          recorder.submitSafely(options, "screenshot_recorder.mask") {
-            val debugMasks = mutableListOf<Rect>()
-
-            val canvas = Canvas(screenshot)
-            canvas.setMatrix(prescaledMatrix)
-            viewHierarchy.traverse { node ->
-              if (node.shouldMask && (node.width > 0 && node.height > 0)) {
-                node.visibleRect ?: return@traverse false
-
-                // TODO: investigate why it returns true on RN when it shouldn't
-                //                                    if (viewHierarchy.isObscured(node)) {
-                //                                        return@traverse true
-                //                                    }
-
-                val (visibleRects, color) =
-                  when (node) {
-                    is ImageViewHierarchyNode -> {
-                      listOf(node.visibleRect) to screenshot.dominantColorForRect(node.visibleRect)
-                    }
-
-                    is TextViewHierarchyNode -> {
-                      val textColor =
-                        node.layout?.dominantTextColor ?: node.dominantColor ?: Color.BLACK
-                      node.layout.getVisibleRects(
-                        node.visibleRect,
-                        node.paddingLeft,
-                        node.paddingTop,
-                      ) to textColor
-                    }
-
-                    else -> {
-                      listOf(node.visibleRect) to Color.BLACK
-                    }
-                  }
-
-                maskingPaint.setColor(color)
-                visibleRects.forEach { rect ->
-                  canvas.drawRoundRect(RectF(rect), 10f, 10f, maskingPaint)
-                }
-                if (options.replayController.isDebugMaskingOverlayEnabled()) {
-                  debugMasks.addAll(visibleRects)
-                }
-              }
-              return@traverse true
-            }
-
-            if (options.replayController.isDebugMaskingOverlayEnabled()) {
-              mainLooperHandler.post {
-                if (debugOverlayDrawable.callback == null) {
-                  root.overlay.add(debugOverlayDrawable)
-                }
-                debugOverlayDrawable.updateMasks(debugMasks)
-                root.postInvalidate()
-              }
-            }
-            screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
-            lastCaptureSuccessful.set(true)
-            contentChanged.set(false)
-          }
-        },
-        mainLooperHandler.handler,
-      )
+      screenshotStrategy.capture(root)
     } catch (e: Throwable) {
       options.logger.log(WARNING, "Failed to capture replay recording", e)
-      lastCaptureSuccessful.set(false)
     }
   }
 
@@ -199,6 +106,7 @@ internal class ScreenshotRecorder(
     }
 
     contentChanged.set(true)
+    screenshotStrategy.onContentChanged()
   }
 
   fun bind(root: View) {
@@ -212,6 +120,7 @@ internal class ScreenshotRecorder(
 
     // invalidate the flag to capture the first frame after new window is attached
     contentChanged.set(true)
+    screenshotStrategy.onContentChanged()
   }
 
   fun unbind(root: View?) {
@@ -235,29 +144,9 @@ internal class ScreenshotRecorder(
   fun close() {
     unbind(rootView?.get())
     rootView?.clear()
-    if (!screenshot.isRecycled) {
-      screenshot.recycle()
-    }
+    screenshotStrategy.close()
     isCapturing.set(false)
   }
-
-  private fun Bitmap.dominantColorForRect(rect: Rect): Int {
-    // TODO: maybe this ceremony can be just simplified to
-    // TODO: multiplying the visibleRect by the prescaledMatrix
-    val visibleRect = Rect(rect)
-    val visibleRectF = RectF(visibleRect)
-
-    // since we take screenshot with lower scale, we also
-    // have to apply the same scale to the visibleRect to get the
-    // correct screenshot part to determine the dominant color
-    prescaledMatrix.mapRect(visibleRectF)
-    // round it back to integer values, because drawBitmap below accepts Rect only
-    visibleRectF.round(visibleRect)
-    // draw part of the screenshot (visibleRect) to a single pixel bitmap
-    singlePixelBitmapCanvas.drawBitmap(this, visibleRect, Rect(0, 0, 1, 1), null)
-    // get the pixel color (= dominant color)
-    return singlePixelBitmap.getPixel(0, 0)
-  }
 }
 
 public data class ScreenshotRecorderConfig(

sentry-android-replay/src/main/java/io/sentry/android/replay/WindowRecorder.kt

@@ -2,6 +2,8 @@ package io.sentry.android.replay
 
 import android.annotation.TargetApi
 import android.graphics.Point
+import android.os.Handler
+import android.os.HandlerThread
 import android.view.View
 import android.view.ViewTreeObserver
 import io.sentry.SentryLevel.DEBUG
@@ -24,18 +26,20 @@ internal class WindowRecorder(
   private val windowCallback: WindowCallback,
   private val mainLooperHandler: MainLooperHandler,
   private val replayExecutor: ScheduledExecutorService,
-) : Recorder, OnRootViewsChangedListener {
-  internal companion object {
-    private const val TAG = "WindowRecorder"
-  }
+) : Recorder, OnRootViewsChangedListener, ExecutorProvider {
 
   private val isRecording = AtomicBoolean(false)
   private val rootViews = ArrayList<WeakReference<View>>()
   private var lastKnownWindowSize: Point = Point()
   private val rootViewsLock = AutoClosableReentrantLock()
   private val capturerLock = AutoClosableReentrantLock()
+  private val backgroundProcessingHandlerLock = AutoClosableReentrantLock()
+
   @Volatile private var capturer: Capturer? = null
 
+  @Volatile private var backgroundProcessingHandlerThread: HandlerThread? = null
+  @Volatile private var backgroundProcessingHandler: Handler? = null
+
   private class Capturer(
     private val options: SentryOptions,
     private val mainLooperHandler: MainLooperHandler,
@@ -177,14 +181,7 @@ internal class WindowRecorder(
     }
 
     capturer?.config = config
-    capturer?.recorder =
-      ScreenshotRecorder(
-        config,
-        options,
-        mainLooperHandler,
-        replayExecutor,
-        screenshotRecorderCallback,
-      )
+    capturer?.recorder = ScreenshotRecorder(config, options, this, screenshotRecorderCallback)
 
     val newRoot = rootViews.lastOrNull()?.get()
     if (newRoot != null) {
@@ -232,6 +229,40 @@ internal class WindowRecorder(
   override fun close() {
     reset()
     mainLooperHandler.removeCallbacks(capturer)
+    backgroundProcessingHandlerLock.acquire().use {
+      backgroundProcessingHandler?.removeCallbacksAndMessages(null)
+      backgroundProcessingHandlerThread?.quitSafely()
+    }
     stop()
   }
+
+  override fun getExecutor(): ScheduledExecutorService = replayExecutor
+
+  override fun getMainLooperHandler(): MainLooperHandler = mainLooperHandler
+
+  override fun getBackgroundHandler(): Handler {
+    // only start the background thread if it's actually needed, as it's only used by Canvas Capture
+    // Strategy
+    if (backgroundProcessingHandler == null) {
+      backgroundProcessingHandlerLock.acquire().use {
+        if (backgroundProcessingHandler == null) {
+          backgroundProcessingHandlerThread = HandlerThread("SentryReplayBackgroundProcessing")
+          backgroundProcessingHandlerThread?.start()
+          backgroundProcessingHandler = Handler(backgroundProcessingHandlerThread!!.looper)
+        }
+      }
+    }
+    return backgroundProcessingHandler!!
+  }
+}
+
+internal interface ExecutorProvider {
+  /** Returns an executor suitable for background tasks. */
+  fun getExecutor(): ScheduledExecutorService
+
+  /** Returns a handler associated with the main thread looper. */
+  fun getMainLooperHandler(): MainLooperHandler
+
+  /** Returns a handler associated with a background thread looper. */
+  fun getBackgroundHandler(): Handler
 }