From 351c5551da748a261a42723960c2bcfe01774c0a Mon Sep 17 00:00:00 2001 From: Eoghan Murray Date: Mon, 13 Feb 2023 17:27:15 +0000 Subject: [PATCH 1/6] Record pointerType on clicks - could be useful for displaying e.g. a circle rather than a point during replay - We have to switch to 'onpointerdown' & 'onpointerup' in order to actually capture `e.pointerType` - this replaces 4 event listeners (MouseDown/MouseUp/TouchStart/TouchEnd) with 2 pointer ones which should fire in all 4 scenarios. We still output the old types according to the MouseInteractions enum - there is no Pointer equivalent of Click, so we leave that is, but use the last Pointer event to attach a pointerType to (only) the click event, where it is most useful - we can fallback to the old method for any browsers not supporting `window.PointerEvent`, in which case \`pointerType\` will be absent from all events --- packages/rrweb/src/record/observer.ts | 49 ++++++++++++++++++++++++--- packages/rrweb/src/utils.ts | 4 +-- packages/types/src/index.ts | 1 + 3 files changed, 47 insertions(+), 7 deletions(-) diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index d2852b970a..01a23218be 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -9,7 +9,7 @@ import { getWindowHeight, getWindowWidth, isBlocked, - isTouchEvent, + legacy_isTouchEvent, patch, StyleSheetMirror, } from '../utils'; @@ -170,7 +170,8 @@ function initMoveObserver({ throttle( callbackWrapper((evt) => { const target = getEventTarget(evt); - const { clientX, clientY } = isTouchEvent(evt) + // 'legacy' here as we could switch to https://developer.mozilla.org/en-US/docs/Web/API/Element/pointermove_event + const { clientX, clientY } = legacy_isTouchEvent(evt) ? evt.changedTouches[0] : evt; if (!timeBaseline) { @@ -228,13 +229,38 @@ function initMouseInteractionObserver({ : sampling.mouseInteraction; const handlers: listenerHandler[] = []; + let currentPointerType = null; const getHandler = (eventKey: keyof typeof MouseInteractions) => { - return (event: MouseEvent | TouchEvent) => { + return (event: MouseEvent | TouchEvent | PointerEvent) => { const target = getEventTarget(event) as Node; if (isBlocked(target, blockClass, blockSelector, true)) { return; } - const e = isTouchEvent(event) ? event.changedTouches[0] : event; + let pointerType = null; + let e = event; + if ('pointerType' in e) { + pointerType = (e as PointerEvent).pointerType; // touch / pen / mouse + if (pointerType === 'touch') { + if (MouseInteractions[eventKey] === MouseInteractions.MouseDown) { + // we are actually listening on 'pointerdown' + eventKey = 'TouchStart'; + } else if (MouseInteractions[eventKey] === MouseInteractions.MouseUp) { + // we are actually listening on 'pointerup' + eventKey = 'TouchEnd'; + } + } else if (pointerType == 'pen') { + // TODO: these will get incorrectly emitted as MouseDown/MouseUp + } + } else if (legacy_isTouchEvent(event)) { + e = event.changedTouches[0]; + pointerType = 'touch'; + } + if (pointerType) { + currentPointerType = pointerType; + } else if (MouseInteractions[eventKey] === MouseInteractions.Click) { + pointerType = currentPointerType; + currentPointerType = null; // cleanup as we've used it + } if (!e) { return; } @@ -245,6 +271,7 @@ function initMouseInteractionObserver({ id, x: clientX, y: clientY, + ...pointerType && { pointerType } }); }; }; @@ -256,8 +283,20 @@ function initMouseInteractionObserver({ disableMap[key] !== false, ) .forEach((eventKey: keyof typeof MouseInteractions) => { - const eventName = eventKey.toLowerCase(); + let eventName = eventKey.toLowerCase(); const handler = getHandler(eventKey); + if (window.PointerEvent) { + switch(MouseInteractions[eventKey]) { + case MouseInteractions.MouseDown: + case MouseInteractions.MouseUp: + eventName = eventName.replace('mouse', 'pointer'); + break; + case MouseInteractions.TouchStart: + case MouseInteractions.TouchEnd: + // these are handled by pointerdown/pointerup + return; + } + } handlers.push(on(eventName, handler, doc)); }); return callbackWrapper(() => { diff --git a/packages/rrweb/src/utils.ts b/packages/rrweb/src/utils.ts index 1626e3734c..78ca08448d 100644 --- a/packages/rrweb/src/utils.ts +++ b/packages/rrweb/src/utils.ts @@ -277,8 +277,8 @@ export function isAncestorRemoved(target: Node, mirror: Mirror): boolean { return isAncestorRemoved(target.parentNode, mirror); } -export function isTouchEvent( - event: MouseEvent | TouchEvent, +export function legacy_isTouchEvent( + event: MouseEvent | TouchEvent | PointerEvent, ): event is TouchEvent { return Boolean((event as TouchEvent).changedTouches); } diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 1188ef2fb7..2c04794829 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -404,6 +404,7 @@ type mouseInteractionParam = { id: number; x: number; y: number; + pointerType?: string; }; export type mouseInteractionCallBack = (d: mouseInteractionParam) => void; From 73ee29f2082169b74d9fa77f111c0799d140ac26 Mon Sep 17 00:00:00 2001 From: eoghanmurray Date: Thu, 30 Mar 2023 16:12:33 +0000 Subject: [PATCH 2/6] Apply formatting changes --- packages/rrweb/src/record/observer.ts | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index 01a23218be..bdfaae546f 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -244,7 +244,9 @@ function initMouseInteractionObserver({ if (MouseInteractions[eventKey] === MouseInteractions.MouseDown) { // we are actually listening on 'pointerdown' eventKey = 'TouchStart'; - } else if (MouseInteractions[eventKey] === MouseInteractions.MouseUp) { + } else if ( + MouseInteractions[eventKey] === MouseInteractions.MouseUp + ) { // we are actually listening on 'pointerup' eventKey = 'TouchEnd'; } @@ -259,7 +261,7 @@ function initMouseInteractionObserver({ currentPointerType = pointerType; } else if (MouseInteractions[eventKey] === MouseInteractions.Click) { pointerType = currentPointerType; - currentPointerType = null; // cleanup as we've used it + currentPointerType = null; // cleanup as we've used it } if (!e) { return; @@ -271,7 +273,7 @@ function initMouseInteractionObserver({ id, x: clientX, y: clientY, - ...pointerType && { pointerType } + ...(pointerType && { pointerType }), }); }; }; @@ -286,15 +288,15 @@ function initMouseInteractionObserver({ let eventName = eventKey.toLowerCase(); const handler = getHandler(eventKey); if (window.PointerEvent) { - switch(MouseInteractions[eventKey]) { - case MouseInteractions.MouseDown: - case MouseInteractions.MouseUp: - eventName = eventName.replace('mouse', 'pointer'); - break; - case MouseInteractions.TouchStart: - case MouseInteractions.TouchEnd: - // these are handled by pointerdown/pointerup - return; + switch (MouseInteractions[eventKey]) { + case MouseInteractions.MouseDown: + case MouseInteractions.MouseUp: + eventName = eventName.replace('mouse', 'pointer'); + break; + case MouseInteractions.TouchStart: + case MouseInteractions.TouchEnd: + // these are handled by pointerdown/pointerup + return; } } handlers.push(on(eventName, handler, doc)); From 979d2b1847a3d05e2731722952e4d6bd8be54f40 Mon Sep 17 00:00:00 2001 From: Eoghan Murray Date: Thu, 30 Mar 2023 17:22:16 +0100 Subject: [PATCH 3/6] Create little-suits-leave.md --- .changeset/little-suits-leave.md | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changeset/little-suits-leave.md diff --git a/.changeset/little-suits-leave.md b/.changeset/little-suits-leave.md new file mode 100644 index 0000000000..3972ff48c0 --- /dev/null +++ b/.changeset/little-suits-leave.md @@ -0,0 +1,6 @@ +--- +"rrweb": minor +"@rrweb/types": minor +--- + +click events (as well as mousedown/mouseup/touchstart/touchend events) now include a `.pointerType` attribute which distinguishes between ['pen', 'mouse' and 'touch' events](https://developer.mozilla.org/en-US/docs/Web/API/PointerEvent/pointerType) From 2114dc47d478b4fa892ac1f87f8eee36680504dd Mon Sep 17 00:00:00 2001 From: eoghanmurray Date: Thu, 30 Mar 2023 16:23:41 +0000 Subject: [PATCH 4/6] Apply formatting changes --- .changeset/little-suits-leave.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.changeset/little-suits-leave.md b/.changeset/little-suits-leave.md index 3972ff48c0..29f44bb5ff 100644 --- a/.changeset/little-suits-leave.md +++ b/.changeset/little-suits-leave.md @@ -1,6 +1,6 @@ --- -"rrweb": minor -"@rrweb/types": minor +'rrweb': minor +'@rrweb/types': minor --- click events (as well as mousedown/mouseup/touchstart/touchend events) now include a `.pointerType` attribute which distinguishes between ['pen', 'mouse' and 'touch' events](https://developer.mozilla.org/en-US/docs/Web/API/PointerEvent/pointerType) From 3a6de4b1292cb62f8ead3856ae30fb295029789e Mon Sep 17 00:00:00 2001 From: Eoghan Murray Date: Wed, 5 Apr 2023 15:06:46 +0100 Subject: [PATCH 5/6] Make the pointerType into an Enum to be consistent with other values in events --- packages/rrweb/src/record/observer.ts | 21 ++++++++++++++------- packages/types/src/index.ts | 8 +++++++- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index bdfaae546f..241c0e5645 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -20,6 +20,7 @@ import { mousePosition, mouseInteractionCallBack, MouseInteractions, + PointerTypes, listenerHandler, scrollCallback, styleSheetRuleCallback, @@ -236,11 +237,17 @@ function initMouseInteractionObserver({ if (isBlocked(target, blockClass, blockSelector, true)) { return; } - let pointerType = null; + let pointerType: PointerTypes | null = null; let e = event; if ('pointerType' in e) { - pointerType = (e as PointerEvent).pointerType; // touch / pen / mouse - if (pointerType === 'touch') { + Object.keys(PointerTypes) + .forEach((pointerKey: keyof typeof PointerKeys) => { + if ((e as PointerEvent).pointerType === pointerKey.toLowerCase()) { + pointerType = PointerTypes[pointerKey]; + return; + } + }); + if (pointerType === PointerTypes.Touch) { if (MouseInteractions[eventKey] === MouseInteractions.MouseDown) { // we are actually listening on 'pointerdown' eventKey = 'TouchStart'; @@ -250,14 +257,14 @@ function initMouseInteractionObserver({ // we are actually listening on 'pointerup' eventKey = 'TouchEnd'; } - } else if (pointerType == 'pen') { + } else if (pointerType == PointerTypes.Pen) { // TODO: these will get incorrectly emitted as MouseDown/MouseUp } } else if (legacy_isTouchEvent(event)) { e = event.changedTouches[0]; - pointerType = 'touch'; + pointerType = PointerTypes.Touch; } - if (pointerType) { + if (pointerType !== null) { currentPointerType = pointerType; } else if (MouseInteractions[eventKey] === MouseInteractions.Click) { pointerType = currentPointerType; @@ -273,7 +280,7 @@ function initMouseInteractionObserver({ id, x: clientX, y: clientY, - ...(pointerType && { pointerType }), + ...(pointerType !== null && { pointerType }), }); }; }; diff --git a/packages/types/src/index.ts b/packages/types/src/index.ts index 2c04794829..6601457291 100644 --- a/packages/types/src/index.ts +++ b/packages/types/src/index.ts @@ -362,6 +362,12 @@ export enum MouseInteractions { TouchCancel, } +export enum PointerTypes { + Mouse, + Pen, + Touch, +} + export enum CanvasContext { '2D', WebGL, @@ -404,7 +410,7 @@ type mouseInteractionParam = { id: number; x: number; y: number; - pointerType?: string; + pointerType?: PointerTypes; }; export type mouseInteractionCallBack = (d: mouseInteractionParam) => void; From be856443dd66eb9068637858dc75851208b02e2d Mon Sep 17 00:00:00 2001 From: eoghanmurray Date: Wed, 5 Apr 2023 15:51:24 +0000 Subject: [PATCH 6/6] Apply formatting changes --- packages/rrweb/src/record/observer.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/packages/rrweb/src/record/observer.ts b/packages/rrweb/src/record/observer.ts index 241c0e5645..2e4ab35976 100644 --- a/packages/rrweb/src/record/observer.ts +++ b/packages/rrweb/src/record/observer.ts @@ -240,13 +240,14 @@ function initMouseInteractionObserver({ let pointerType: PointerTypes | null = null; let e = event; if ('pointerType' in e) { - Object.keys(PointerTypes) - .forEach((pointerKey: keyof typeof PointerKeys) => { + Object.keys(PointerTypes).forEach( + (pointerKey: keyof typeof PointerKeys) => { if ((e as PointerEvent).pointerType === pointerKey.toLowerCase()) { pointerType = PointerTypes[pointerKey]; return; } - }); + }, + ); if (pointerType === PointerTypes.Touch) { if (MouseInteractions[eventKey] === MouseInteractions.MouseDown) { // we are actually listening on 'pointerdown'