Skip to content

Commit b287001

Browse files
romtsnclaude
romtsn
and
claude
committed
feat(replay): Capture SurfaceView content (experimental)
SurfaceView (used by Unity, video players, maps, and similar) renders to a separate Surface that is composited by SurfaceFlinger outside of the View hierarchy. PixelCopy.request(window, ...) only captures the Window surface, so SurfaceView regions appeared as transparent/black holes in Session Replay recordings. When the experimental option options.sessionReplay.isCaptureSurfaceViews is enabled, each visible SurfaceView is now captured separately via PixelCopy.request(surfaceView, ...) and composited onto the screenshot using PorterDuff.DST_OVER, so the SurfaceView content draws behind the Window content (which has transparent holes where the SurfaceViews are). Because SurfaceView redraws do not trigger ViewTreeObserver.OnDrawListener, the recorder bypasses the contentChanged guard when SurfaceViews are present, so subsequent frames are re-captured at the configured frame rate instead of reusing the last screenshot. The option defaults to false to preserve existing behavior. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
1 parent 6b019b7 commit b287001

7 files changed

Lines changed: 252 additions & 25 deletions

File tree

‎CHANGELOG.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,11 @@
22

33
## 8.40.0
44

5+
### Features
6+
7+
- Session Replay: experimental support for capturing `SurfaceView` content (e.g. Unity, video players, maps) ([#5333](https://github.com/getsentry/sentry-java/pull/5333))
8+
- To enable, set `options.sessionReplay.isCaptureSurfaceViews = true`
9+
510
### Fixes
611

712
- Fix `NoSuchMethodError` for `LayoutCoordinates.localBoundingBoxOf$default` on Compose touch dispatch with AGP 8.13 and `minSdk < 24` ([#5302](https://github.com/getsentry/sentry-java/pull/5302))

‎sentry-android-replay/src/main/java/io/sentry/android/replay/ScreenshotRecorder.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,7 @@ internal class ScreenshotRecorder(
7070
)
7171
}
7272

73-
if (!contentChanged.get()) {
73+
if (!contentChanged.get() && !screenshotStrategy.hasSurfaceViews()) {
7474
screenshotStrategy.emitLastScreenshot()
7575
return
7676
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/screenshot/PixelCopyStrategy.kt‎

Lines changed: 153 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,13 @@ package io.sentry.android.replay.screenshot
22

33
import android.annotation.SuppressLint
44
import android.graphics.Bitmap
5+
import android.graphics.Canvas
56
import android.graphics.Matrix
7+
import android.graphics.Paint
8+
import android.graphics.PorterDuff
9+
import android.graphics.PorterDuffXfermode
10+
import android.graphics.Rect
11+
import android.graphics.RectF
612
import android.view.PixelCopy
713
import android.view.View
814
import io.sentry.SentryLevel.DEBUG
@@ -19,6 +25,7 @@ import io.sentry.android.replay.util.ReplayRunnable
1925
import io.sentry.android.replay.util.traverse
2026
import io.sentry.android.replay.viewhierarchy.ViewHierarchyNode
2127
import java.util.concurrent.atomic.AtomicBoolean
28+
import java.util.concurrent.atomic.AtomicInteger
2229
import kotlin.LazyThreadSafetyMode.NONE
2330

2431
@SuppressLint("UseKtx")
@@ -40,6 +47,17 @@ internal class PixelCopyStrategy(
4047
private val maskRenderer = MaskRenderer()
4148
private val contentChanged = AtomicBoolean(false)
4249
private val isClosed = AtomicBoolean(false)
50+
private val hasSurfaceViews = AtomicBoolean(false)
51+
private val dstOverPaint by lazy(NONE) {
52+
Paint().apply { xfermode = PorterDuffXfermode(PorterDuff.Mode.DST_OVER) }
53+
}
54+
private val screenshotCanvas by lazy(NONE) { Canvas(screenshot) }
55+
private val tmpSrcRect = Rect()
56+
private val tmpDstRect = RectF()
57+
private val windowLocation = IntArray(2)
58+
private val svLocation = IntArray(2)
59+
60+
private class SurfaceViewCapture(val bitmap: Bitmap, val x: Int, val y: Int)
4361

4462
@SuppressLint("NewApi")
4563
override fun capture(root: View) {
@@ -81,31 +99,22 @@ internal class PixelCopyStrategy(
8199

82100
// TODO: disableAllMasking here and dont traverse?
83101
val viewHierarchy = ViewHierarchyNode.fromView(root, null, 0, options.sessionReplay)
84-
root.traverse(viewHierarchy, options.sessionReplay, options.logger)
85-
86-
executor.submit(
87-
ReplayRunnable("screenshot_recorder.mask") {
88-
if (isClosed.get() || screenshot.isRecycled) {
89-
options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping masking")
90-
return@ReplayRunnable
91-
}
102+
val captureSurfaceViewsEnabled = options.sessionReplay.isCaptureSurfaceViews
103+
val surfaceViewNodes =
104+
if (captureSurfaceViewsEnabled) {
105+
mutableListOf<ViewHierarchyNode.SurfaceViewHierarchyNode>()
106+
} else {
107+
null
108+
}
109+
root.traverse(viewHierarchy, options.sessionReplay, options.logger, surfaceViewNodes)
92110

93-
val debugMasks = maskRenderer.renderMasks(screenshot, viewHierarchy, prescaledMatrix)
111+
hasSurfaceViews.set(surfaceViewNodes?.isNotEmpty() == true)
94112

95-
if (options.replayController.isDebugMaskingOverlayEnabled()) {
96-
mainLooperHandler.post {
97-
if (debugOverlayDrawable.callback == null) {
98-
root.overlay.add(debugOverlayDrawable)
99-
}
100-
debugOverlayDrawable.updateMasks(debugMasks)
101-
root.postInvalidate()
102-
}
103-
}
104-
screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
105-
lastCaptureSuccessful.set(true)
106-
contentChanged.set(false)
107-
}
108-
)
113+
if (surfaceViewNodes.isNullOrEmpty()) {
114+
submitMaskingAndCallback(root, viewHierarchy)
115+
} else {
116+
captureSurfaceViews(root, surfaceViewNodes, viewHierarchy)
117+
}
109118
},
110119
mainLooperHandler.handler,
111120
)
@@ -115,6 +124,123 @@ internal class PixelCopyStrategy(
115124
}
116125
}
117126

127+
private fun submitMaskingAndCallback(root: View, viewHierarchy: ViewHierarchyNode) {
128+
executor.submit(
129+
ReplayRunnable("screenshot_recorder.mask") { applyMaskingAndNotify(root, viewHierarchy) }
130+
)
131+
}
132+
133+
private fun applyMaskingAndNotify(root: View, viewHierarchy: ViewHierarchyNode) {
134+
if (isClosed.get() || screenshot.isRecycled) {
135+
options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping masking")
136+
return
137+
}
138+
139+
val debugMasks = maskRenderer.renderMasks(screenshot, viewHierarchy, prescaledMatrix)
140+
141+
if (options.replayController.isDebugMaskingOverlayEnabled()) {
142+
mainLooperHandler.post {
143+
if (debugOverlayDrawable.callback == null) {
144+
root.overlay.add(debugOverlayDrawable)
145+
}
146+
debugOverlayDrawable.updateMasks(debugMasks)
147+
root.postInvalidate()
148+
}
149+
}
150+
screenshotRecorderCallback?.onScreenshotRecorded(screenshot)
151+
lastCaptureSuccessful.set(true)
152+
contentChanged.set(false)
153+
}
154+
155+
@SuppressLint("NewApi")
156+
private fun captureSurfaceViews(
157+
root: View,
158+
surfaceViewNodes: List<ViewHierarchyNode.SurfaceViewHierarchyNode>,
159+
viewHierarchy: ViewHierarchyNode,
160+
) {
161+
root.getLocationOnScreen(windowLocation)
162+
163+
val captures = arrayOfNulls<SurfaceViewCapture>(surfaceViewNodes.size)
164+
val remaining = AtomicInteger(surfaceViewNodes.size)
165+
166+
fun onCaptureComplete() {
167+
if (remaining.decrementAndGet() == 0) {
168+
compositeSurfaceViewsAndMask(root, captures, viewHierarchy)
169+
}
170+
}
171+
172+
for ((index, node) in surfaceViewNodes.withIndex()) {
173+
val surfaceView = node.surfaceViewRef.get()
174+
if (surfaceView == null || !surfaceView.holder.surface.isValid) {
175+
onCaptureComplete()
176+
continue
177+
}
178+
179+
try {
180+
val svBitmap =
181+
Bitmap.createBitmap(surfaceView.width, surfaceView.height, Bitmap.Config.ARGB_8888)
182+
183+
surfaceView.getLocationOnScreen(svLocation)
184+
val capturedX = svLocation[0]
185+
val capturedY = svLocation[1]
186+
187+
PixelCopy.request(
188+
surfaceView,
189+
svBitmap,
190+
{ copyResult: Int ->
191+
if (copyResult == PixelCopy.SUCCESS) {
192+
captures[index] = SurfaceViewCapture(svBitmap, capturedX, capturedY)
193+
} else {
194+
svBitmap.recycle()
195+
options.logger.log(INFO, "Failed to capture SurfaceView: %d", copyResult)
196+
}
197+
onCaptureComplete()
198+
},
199+
mainLooperHandler.handler,
200+
)
201+
} catch (e: Throwable) {
202+
options.logger.log(WARNING, "Failed to capture SurfaceView", e)
203+
onCaptureComplete()
204+
}
205+
}
206+
}
207+
208+
private fun compositeSurfaceViewsAndMask(
209+
root: View,
210+
captures: Array<SurfaceViewCapture?>,
211+
viewHierarchy: ViewHierarchyNode,
212+
) {
213+
executor.submit(
214+
ReplayRunnable("screenshot_recorder.composite") {
215+
if (isClosed.get() || screenshot.isRecycled) {
216+
options.logger.log(DEBUG, "PixelCopyStrategy is closed, skipping compositing")
217+
return@ReplayRunnable
218+
}
219+
220+
for (capture in captures) {
221+
if (capture == null) continue
222+
if (capture.bitmap.isRecycled) continue
223+
224+
val left = (capture.x - windowLocation[0]) * config.scaleFactorX
225+
val top = (capture.y - windowLocation[1]) * config.scaleFactorY
226+
tmpSrcRect.set(0, 0, capture.bitmap.width, capture.bitmap.height)
227+
tmpDstRect.set(
228+
left,
229+
top,
230+
left + capture.bitmap.width * config.scaleFactorX,
231+
top + capture.bitmap.height * config.scaleFactorY,
232+
)
233+
234+
// DST_OVER draws the SurfaceView content behind the existing Window content
235+
screenshotCanvas.drawBitmap(capture.bitmap, tmpSrcRect, tmpDstRect, dstOverPaint)
236+
capture.bitmap.recycle()
237+
}
238+
239+
applyMaskingAndNotify(root, viewHierarchy)
240+
}
241+
)
242+
}
243+
118244
override fun onContentChanged() {
119245
contentChanged.set(true)
120246
}
@@ -123,6 +249,10 @@ internal class PixelCopyStrategy(
123249
return lastCaptureSuccessful.get()
124250
}
125251

252+
override fun hasSurfaceViews(): Boolean {
253+
return hasSurfaceViews.get()
254+
}
255+
126256
override fun emitLastScreenshot() {
127257
if (lastCaptureSuccessful() && !screenshot.isRecycled) {
128258
screenshotRecorderCallback?.onScreenshotRecorded(screenshot)

‎sentry-android-replay/src/main/java/io/sentry/android/replay/screenshot/ScreenshotStrategy.kt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,11 @@ internal interface ScreenshotStrategy {
1212
fun lastCaptureSuccessful(): Boolean
1313

1414
fun emitLastScreenshot()
15+
16+
/**
17+
* Whether the last capture detected SurfaceViews that render independently of the View tree. When
18+
* true, the recorder bypasses the contentChanged guard since SurfaceView redraws don't trigger
19+
* ViewTreeObserver.OnDrawListener.
20+
*/
21+
fun hasSurfaceViews(): Boolean = false
1522
}

‎sentry-android-replay/src/main/java/io/sentry/android/replay/util/Views.kt‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,12 @@ import java.lang.NullPointerException
3434
* @param logger Logger for error reporting during Compose traversal
3535
*/
3636
@SuppressLint("UseKtx")
37+
@JvmOverloads
3738
internal fun View.traverse(
3839
parentNode: ViewHierarchyNode,
3940
options: SentryMaskingOptions,
4041
logger: ILogger,
42+
surfaceViewNodes: MutableList<ViewHierarchyNode.SurfaceViewHierarchyNode>? = null,
4143
) {
4244
if (this !is ViewGroup) {
4345
return
@@ -59,7 +61,14 @@ internal fun View.traverse(
5961
if (child != null) {
6062
val childNode = ViewHierarchyNode.fromView(child, parentNode, indexOfChild(child), options)
6163
childNodes.add(childNode)
62-
child.traverse(childNode, options, logger)
64+
if (
65+
surfaceViewNodes != null &&
66+
childNode is ViewHierarchyNode.SurfaceViewHierarchyNode &&
67+
childNode.isVisible
68+
) {
69+
surfaceViewNodes.add(childNode)
70+
}
71+
child.traverse(childNode, options, logger, surfaceViewNodes)
6372
}
6473
}
6574
parentNode.children = childNodes

‎sentry-android-replay/src/main/java/io/sentry/android/replay/viewhierarchy/ViewHierarchyNode.kt‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package io.sentry.android.replay.viewhierarchy
33
import android.annotation.SuppressLint
44
import android.annotation.TargetApi
55
import android.graphics.Rect
6+
import android.view.SurfaceView
67
import android.view.View
78
import android.view.ViewParent
89
import android.widget.ImageView
@@ -15,6 +16,7 @@ import io.sentry.android.replay.util.isMaskable
1516
import io.sentry.android.replay.util.isVisibleToUser
1617
import io.sentry.android.replay.util.toOpaque
1718
import io.sentry.android.replay.util.totalPaddingTopSafe
19+
import java.lang.ref.WeakReference
1820

1921
@SuppressLint("UseRequiresApi")
2022
@TargetApi(26)
@@ -121,6 +123,34 @@ internal sealed class ViewHierarchyNode(
121123
visibleRect,
122124
)
123125

126+
class SurfaceViewHierarchyNode(
127+
val surfaceViewRef: WeakReference<SurfaceView>,
128+
x: Float,
129+
y: Float,
130+
width: Int,
131+
height: Int,
132+
elevation: Float,
133+
distance: Int,
134+
parent: ViewHierarchyNode? = null,
135+
shouldMask: Boolean = false,
136+
isImportantForContentCapture: Boolean = false,
137+
isVisible: Boolean = false,
138+
visibleRect: Rect? = null,
139+
) :
140+
ViewHierarchyNode(
141+
x,
142+
y,
143+
width,
144+
height,
145+
elevation,
146+
distance,
147+
parent,
148+
shouldMask,
149+
isImportantForContentCapture,
150+
isVisible,
151+
visibleRect,
152+
)
153+
124154
/**
125155
* Basically replicating this:
126156
* https://developer.android.com/reference/android/view/View#isImportantForContentCapture() but
@@ -379,6 +409,24 @@ internal sealed class ViewHierarchyNode(
379409
visibleRect = visibleRect,
380410
)
381411
}
412+
413+
is SurfaceView -> {
414+
parent?.setImportantForCaptureToAncestors(true)
415+
return SurfaceViewHierarchyNode(
416+
surfaceViewRef = WeakReference(view),
417+
x = view.x,
418+
y = view.y,
419+
width = view.width,
420+
height = view.height,
421+
elevation = (parent?.elevation ?: 0f) + view.elevation,
422+
distance = distance,
423+
parent = parent,
424+
shouldMask = shouldMask,
425+
isImportantForContentCapture = true,
426+
isVisible = isVisible,
427+
visibleRect = visibleRect,
428+
)
429+
}
382430
}
383431

384432
return GenericViewHierarchyNode(

0 commit comments

Comments
 (0)