Skip to content

Commit cfc52d9

Browse files
romtsnromtsn
authored
[SR] Support multi-touch gestures
2 parents a83b5d9 + c832416 commit cfc52d9

File tree

11 files changed

+147
-38
lines changed

11 files changed

+147
-38
lines changed

‎sentry-android-replay/src/main/java/io/sentry/android/replay/DefaultReplayBreadcrumbConverter.kt‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ public open class DefaultReplayBreadcrumbConverter : ReplayBreadcrumbConverter {
2222
)
2323
}
2424

25+
private var lastConnectivityState: String? = null
26+
2527
override fun convert(breadcrumb: Breadcrumb): RRWebEvent? {
2628
var breadcrumbMessage: String? = null
2729
var breadcrumbCategory: String? = null
@@ -79,6 +81,13 @@ public open class DefaultReplayBreadcrumbConverter : ReplayBreadcrumbConverter {
7981

8082
else -> return null
8183
}
84+
85+
if (lastConnectivityState == breadcrumbData["state"]) {
86+
// debounce same state
87+
return null
88+
}
89+
90+
lastConnectivityState = breadcrumbData["state"] as? String
8291
}
8392

8493
breadcrumb.data["action"] == "BATTERY_CHANGED" -> {

‎sentry-android-replay/src/main/java/io/sentry/android/replay/capture/BaseCaptureStrategy.kt‎

Lines changed: 95 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ internal abstract class BaseCaptureStrategy(
6060

6161
protected val currentEvents = LinkedList<RRWebEvent>()
6262
private val currentEventsLock = Any()
63-
private val currentPositions = mutableListOf<Position>()
63+
private val currentPositions = LinkedHashMap<Int, ArrayList<Position>>(10)
6464
private var touchMoveBaseline = 0L
6565
private var lastCapturedMoveEvent = 0L
6666

@@ -227,10 +227,10 @@ internal abstract class BaseCaptureStrategy(
227227
}
228228

229229
override fun onTouchEvent(event: MotionEvent) {
230-
val rrwebEvent = event.toRRWebIncrementalSnapshotEvent()
231-
if (rrwebEvent != null) {
230+
val rrwebEvents = event.toRRWebIncrementalSnapshotEvent()
231+
if (rrwebEvents != null) {
232232
synchronized(currentEventsLock) {
233-
currentEvents += rrwebEvent
233+
currentEvents += rrwebEvents
234234
}
235235
}
236236
}
@@ -284,9 +284,9 @@ internal abstract class BaseCaptureStrategy(
284284
}
285285
}
286286

287-
private fun MotionEvent.toRRWebIncrementalSnapshotEvent(): RRWebIncrementalSnapshotEvent? {
287+
private fun MotionEvent.toRRWebIncrementalSnapshotEvent(): List<RRWebIncrementalSnapshotEvent>? {
288288
val event = this
289-
return when (val action = event.actionMasked) {
289+
return when (event.actionMasked) {
290290
MotionEvent.ACTION_MOVE -> {
291291
// we only throttle move events as those can be overwhelming
292292
val now = dateProvider.currentTimeMillis
@@ -295,48 +295,109 @@ internal abstract class BaseCaptureStrategy(
295295
}
296296
lastCapturedMoveEvent = now
297297

298-
// idk why but rrweb does it like dis
299-
if (touchMoveBaseline == 0L) {
300-
touchMoveBaseline = now
301-
}
298+
currentPositions.keys.forEach { pId ->
299+
val pIndex = event.findPointerIndex(pId)
300+
301+
if (pIndex == -1) {
302+
// no data for this pointer
303+
return@forEach
304+
}
305+
306+
// idk why but rrweb does it like dis
307+
if (touchMoveBaseline == 0L) {
308+
touchMoveBaseline = now
309+
}
302310

303-
currentPositions += Position().apply {
304-
x = event.x * recorderConfig.scaleFactorX
305-
y = event.y * recorderConfig.scaleFactorY
306-
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
307-
timeOffset = now - touchMoveBaseline
311+
currentPositions[pId]!! += Position().apply {
312+
x = event.getX(pIndex) * recorderConfig.scaleFactorX
313+
y = event.getY(pIndex) * recorderConfig.scaleFactorY
314+
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
315+
timeOffset = now - touchMoveBaseline
316+
}
308317
}
309318

310319
val totalOffset = now - touchMoveBaseline
311320
return if (totalOffset > CAPTURE_MOVE_EVENT_THRESHOLD) {
312-
RRWebInteractionMoveEvent().apply {
313-
timestamp = now
314-
positions = currentPositions.map { pos ->
315-
pos.timeOffset -= totalOffset
316-
pos
321+
val moveEvents = mutableListOf<RRWebInteractionMoveEvent>()
322+
for ((pointerId, positions) in currentPositions) {
323+
if (positions.isNotEmpty()) {
324+
moveEvents += RRWebInteractionMoveEvent().apply {
325+
this.timestamp = now
326+
this.positions = positions.map { pos ->
327+
pos.timeOffset -= totalOffset
328+
pos
329+
}
330+
this.pointerId = pointerId
331+
}
332+
currentPositions[pointerId]!!.clear()
317333
}
318-
}.also {
319-
currentPositions.clear()
320-
touchMoveBaseline = 0L
321334
}
335+
touchMoveBaseline = 0L
336+
moveEvents
322337
} else {
323338
null
324339
}
325340
}
326341

327-
MotionEvent.ACTION_DOWN, MotionEvent.ACTION_UP, MotionEvent.ACTION_CANCEL -> {
328-
RRWebInteractionEvent().apply {
329-
timestamp = dateProvider.currentTimeMillis
330-
x = event.x * recorderConfig.scaleFactorX
331-
y = event.y * recorderConfig.scaleFactorY
332-
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
333-
interactionType = when (action) {
334-
MotionEvent.ACTION_UP -> InteractionType.TouchEnd
335-
MotionEvent.ACTION_DOWN -> InteractionType.TouchStart
336-
MotionEvent.ACTION_CANCEL -> InteractionType.TouchCancel
337-
else -> InteractionType.TouchMove_Departed // should not happen
342+
MotionEvent.ACTION_DOWN,
343+
MotionEvent.ACTION_POINTER_DOWN -> {
344+
val pId = event.getPointerId(event.actionIndex)
345+
val pIndex = event.findPointerIndex(pId)
346+
347+
if (pIndex == -1) {
348+
// no data for this pointer
349+
return null
350+
}
351+
352+
// new finger down - add a new pointer for tracking movement
353+
currentPositions[pId] = ArrayList()
354+
listOf(
355+
RRWebInteractionEvent().apply {
356+
timestamp = dateProvider.currentTimeMillis
357+
x = event.getX(pIndex) * recorderConfig.scaleFactorX
358+
y = event.getY(pIndex) * recorderConfig.scaleFactorY
359+
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
360+
pointerId = pId
361+
interactionType = InteractionType.TouchStart
338362
}
363+
)
364+
}
365+
MotionEvent.ACTION_UP,
366+
MotionEvent.ACTION_POINTER_UP -> {
367+
val pId = event.getPointerId(event.actionIndex)
368+
val pIndex = event.findPointerIndex(pId)
369+
370+
if (pIndex == -1) {
371+
// no data for this pointer
372+
return null
339373
}
374+
375+
// finger lift up - remove the pointer from tracking
376+
currentPositions.remove(pId)
377+
listOf(
378+
RRWebInteractionEvent().apply {
379+
timestamp = dateProvider.currentTimeMillis
380+
x = event.getX(pIndex) * recorderConfig.scaleFactorX
381+
y = event.getY(pIndex) * recorderConfig.scaleFactorY
382+
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
383+
pointerId = pId
384+
interactionType = InteractionType.TouchEnd
385+
}
386+
)
387+
}
388+
MotionEvent.ACTION_CANCEL -> {
389+
// gesture cancelled - remove all pointers from tracking
390+
currentPositions.clear()
391+
listOf(
392+
RRWebInteractionEvent().apply {
393+
timestamp = dateProvider.currentTimeMillis
394+
x = event.x * recorderConfig.scaleFactorX
395+
y = event.y * recorderConfig.scaleFactorY
396+
id = 0 // html node id, but we don't have it, so hardcode to 0 to align with FE
397+
pointerId = 0 // the pointerId is not used for TouchCancel, so just set it to 0
398+
interactionType = InteractionType.TouchCancel
399+
}
400+
)
340401
}
341402

342403
else -> null

‎sentry-android-replay/src/main/java/io/sentry/android/replay/capture/SessionCaptureStrategy.kt‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,7 +129,7 @@ internal class SessionCaptureStrategy(
129129
val now = dateProvider.currentTimeMillis
130130
val currentSegmentTimestamp = segmentTimestamp.get()
131131
val segmentId = currentSegment.get()
132-
val duration = now - currentSegmentTimestamp.time
132+
val duration = now - (currentSegmentTimestamp?.time ?: 0)
133133
val replayId = currentReplayId.get()
134134
val height = recorderConfig.recordingHeight
135135
val width = recorderConfig.recordingWidth

‎sentry/api/sentry.api‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5210,6 +5210,7 @@ public final class io/sentry/rrweb/RRWebInteractionEvent : io/sentry/rrweb/RRWeb
52105210
public fun getDataUnknown ()Ljava/util/Map;
52115211
public fun getId ()I
52125212
public fun getInteractionType ()Lio/sentry/rrweb/RRWebInteractionEvent$InteractionType;
5213+
public fun getPointerId ()I
52135214
public fun getPointerType ()I
52145215
public fun getUnknown ()Ljava/util/Map;
52155216
public fun getX ()F
@@ -5218,6 +5219,7 @@ public final class io/sentry/rrweb/RRWebInteractionEvent : io/sentry/rrweb/RRWeb
52185219
public fun setDataUnknown (Ljava/util/Map;)V
52195220
public fun setId (I)V
52205221
public fun setInteractionType (Lio/sentry/rrweb/RRWebInteractionEvent$InteractionType;)V
5222+
public fun setPointerId (I)V
52215223
public fun setPointerType (I)V
52225224
public fun setUnknown (Ljava/util/Map;)V
52235225
public fun setX (F)V
@@ -5256,6 +5258,7 @@ public final class io/sentry/rrweb/RRWebInteractionEvent$InteractionType$Deseria
52565258
public final class io/sentry/rrweb/RRWebInteractionEvent$JsonKeys {
52575259
public static final field DATA Ljava/lang/String;
52585260
public static final field ID Ljava/lang/String;
5261+
public static final field POINTER_ID Ljava/lang/String;
52595262
public static final field POINTER_TYPE Ljava/lang/String;
52605263
public static final field TYPE Ljava/lang/String;
52615264
public static final field X Ljava/lang/String;
@@ -5266,10 +5269,12 @@ public final class io/sentry/rrweb/RRWebInteractionEvent$JsonKeys {
52665269
public final class io/sentry/rrweb/RRWebInteractionMoveEvent : io/sentry/rrweb/RRWebIncrementalSnapshotEvent, io/sentry/JsonSerializable, io/sentry/JsonUnknown {
52675270
public fun <init> ()V
52685271
public fun getDataUnknown ()Ljava/util/Map;
5272+
public fun getPointerId ()I
52695273
public fun getPositions ()Ljava/util/List;
52705274
public fun getUnknown ()Ljava/util/Map;
52715275
public fun serialize (Lio/sentry/ObjectWriter;Lio/sentry/ILogger;)V
52725276
public fun setDataUnknown (Ljava/util/Map;)V
5277+
public fun setPointerId (I)V
52735278
public fun setPositions (Ljava/util/List;)V
52745279
public fun setUnknown (Ljava/util/Map;)V
52755280
}
@@ -5282,6 +5287,7 @@ public final class io/sentry/rrweb/RRWebInteractionMoveEvent$Deserializer : io/s
52825287

52835288
public final class io/sentry/rrweb/RRWebInteractionMoveEvent$JsonKeys {
52845289
public static final field DATA Ljava/lang/String;
5290+
public static final field POINTER_ID Ljava/lang/String;
52855291
public static final field POSITIONS Ljava/lang/String;
52865292
public fun <init> ()V
52875293
}

‎sentry/src/main/java/io/sentry/rrweb/RRWebInteractionEvent.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,8 @@ public static final class Deserializer implements JsonDeserializer<InteractionTy
5757

5858
private int pointerType = POINTER_TYPE_TOUCH;
5959

60+
private int pointerId;
61+
6062
// to support unknown json attributes with nesting, we have to have unknown map for each of the
6163
// nested object in json: { ..., "data": { ... } }
6264
private @Nullable Map<String, Object> unknown;
@@ -107,6 +109,14 @@ public void setPointerType(final int pointerType) {
107109
this.pointerType = pointerType;
108110
}
109111

112+
public int getPointerId() {
113+
return pointerId;
114+
}
115+
116+
public void setPointerId(final int pointerId) {
117+
this.pointerId = pointerId;
118+
}
119+
110120
@Nullable
111121
public Map<String, Object> getDataUnknown() {
112122
return dataUnknown;
@@ -136,6 +146,7 @@ public static final class JsonKeys {
136146
public static final String X = "x";
137147
public static final String Y = "y";
138148
public static final String POINTER_TYPE = "pointerType";
149+
public static final String POINTER_ID = "pointerId";
139150
}
140151

141152
@Override
@@ -163,6 +174,7 @@ private void serializeData(final @NotNull ObjectWriter writer, final @NotNull IL
163174
writer.name(JsonKeys.X).value(x);
164175
writer.name(JsonKeys.Y).value(y);
165176
writer.name(JsonKeys.POINTER_TYPE).value(pointerType);
177+
writer.name(JsonKeys.POINTER_ID).value(pointerId);
166178
if (dataUnknown != null) {
167179
for (String key : dataUnknown.keySet()) {
168180
Object value = dataUnknown.get(key);
@@ -235,6 +247,9 @@ private void deserializeData(
235247
case JsonKeys.POINTER_TYPE:
236248
event.pointerType = reader.nextInt();
237249
break;
250+
case JsonKeys.POINTER_ID:
251+
event.pointerId = reader.nextInt();
252+
break;
238253
default:
239254
if (!baseEventDeserializer.deserializeValue(event, nextName, reader, logger)) {
240255
if (dataUnknown == null) {

‎sentry/src/main/java/io/sentry/rrweb/RRWebInteractionMoveEvent.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,7 @@ public static final class Deserializer implements JsonDeserializer<Position> {
142142
// endregion json
143143
}
144144

145+
private int pointerId;
145146
private @Nullable List<Position> positions;
146147
// to support unknown json attributes with nesting, we have to have unknown map for each of the
147148
// nested object in json: { ..., "data": { ... } }
@@ -180,12 +181,21 @@ public void setPositions(final @Nullable List<Position> positions) {
180181
this.positions = positions;
181182
}
182183

184+
public int getPointerId() {
185+
return pointerId;
186+
}
187+
188+
public void setPointerId(final int pointerId) {
189+
this.pointerId = pointerId;
190+
}
191+
183192
// region json
184193

185194
// rrweb uses camelCase hence the json keys are in camelCase here
186195
public static final class JsonKeys {
187196
public static final String DATA = "data";
188197
public static final String POSITIONS = "positions";
198+
public static final String POINTER_ID = "pointerId";
189199
}
190200

191201
@Override
@@ -211,6 +221,7 @@ private void serializeData(final @NotNull ObjectWriter writer, final @NotNull IL
211221
if (positions != null && !positions.isEmpty()) {
212222
writer.name(JsonKeys.POSITIONS).value(logger, positions);
213223
}
224+
writer.name(JsonKeys.POINTER_ID).value(pointerId);
214225
if (dataUnknown != null) {
215226
for (String key : dataUnknown.keySet()) {
216227
Object value = dataUnknown.get(key);
@@ -271,6 +282,9 @@ private void deserializeData(
271282
case JsonKeys.POSITIONS:
272283
event.positions = reader.nextListOrNull(logger, new Position.Deserializer());
273284
break;
285+
case JsonKeys.POINTER_ID:
286+
event.pointerId = reader.nextInt();
287+
break;
274288
default:
275289
if (!baseEventDeserializer.deserializeValue(event, nextName, reader, logger)) {
276290
if (dataUnknown == null) {

‎sentry/src/test/java/io/sentry/rrweb/RRWebInteractionEventSerializationTest.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ class RRWebInteractionEventSerializationTest {
1717
x = 1.0f
1818
y = 2.0f
1919
interactionType = TouchStart
20+
pointerId = 1
2021
}
2122
}
2223

‎sentry/src/test/java/io/sentry/rrweb/RRWebInteractionMoveEventSerializationTest.kt‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ class RRWebInteractionMoveEventSerializationTest {
2121
timeOffset = 100
2222
}
2323
)
24+
pointerId = 1
2425
}
2526
}
2627

‎sentry/src/test/resources/json/replay_recording.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
11
{"segment_id":0}
2-
[{"type":4,"timestamp":1234567890,"data":{"href":"https://sentry.io","height":1920,"width":1080}},{"type":5,"timestamp":12345678901,"data":{"tag":"video","payload":{"segmentId":0,"size":4000000,"duration":5000,"encoding":"h264","container":"mp4","height":1920,"width":1080,"frameCount":5,"frameRate":1,"frameRateType":"constant","left":100,"top":100}}},{"type":5,"timestamp":12345678901,"data":{"tag":"breadcrumb","payload":{"type":"default","timestamp":12345678.901,"category":"navigation","message":"message","level":"info","data":{"screen":"MainActivity","state":"resumed"}}}},{"type":5,"timestamp":12345678901,"data":{"tag":"performanceSpan","payload":{"op":"resource.http","description":"https://api.github.com/users/getsentry/repos","startTimestamp":12345678.901,"endTimestamp":12345679.901,"data":{"status_code":200,"method":"POST"}}}},{"type":3,"timestamp":12345678901,"data":{"source":2,"type":7,"id":1,"x":1.0,"y":2.0,"pointerType":2}},{"type":3,"timestamp":12345678901,"data":{"source":6,"positions":[{"id":1,"x":1.0,"y":2.0,"timeOffset":100}]}}]
2+
[{"type":4,"timestamp":1234567890,"data":{"href":"https://sentry.io","height":1920,"width":1080}},{"type":5,"timestamp":12345678901,"data":{"tag":"video","payload":{"segmentId":0,"size":4000000,"duration":5000,"encoding":"h264","container":"mp4","height":1920,"width":1080,"frameCount":5,"frameRate":1,"frameRateType":"constant","left":100,"top":100}}},{"type":5,"timestamp":12345678901,"data":{"tag":"breadcrumb","payload":{"type":"default","timestamp":12345678.901,"category":"navigation","message":"message","level":"info","data":{"screen":"MainActivity","state":"resumed"}}}},{"type":5,"timestamp":12345678901,"data":{"tag":"performanceSpan","payload":{"op":"resource.http","description":"https://api.github.com/users/getsentry/repos","startTimestamp":12345678.901,"endTimestamp":12345679.901,"data":{"status_code":200,"method":"POST"}}}},{"type":3,"timestamp":12345678901,"data":{"source":2,"type":7,"id":1,"x":1.0,"y":2.0,"pointerType":2,"pointerId":1}},{"type":3,"timestamp":12345678901,"data":{"source":6,"positions":[{"id":1,"x":1.0,"y":2.0,"timeOffset":100}],"pointerId":1}}]

‎sentry/src/test/resources/json/rrweb_interaction_event.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"id": 1,
88
"x": 1.0,
99
"y": 2.0,
10-
"pointerType": 2
10+
"pointerType": 2,
11+
"pointerId": 1
1112
}
1213
}

0 commit comments

Comments
 (0)