Canvas recording: Preserve drawing buffer (#1273)

* Upgrade jest to 29 and puppeteer to 16 in rrweb

* Apply formatting changes

* Upgrade rrweb's puppeteer to v20

* Apply formatting changes

* Canvas: Reduce flickering and capturing of empty canvas elements

Turn on `preserveDrawingBuffer` by default for canvas FPS recording.
Has some negative performance implications, but really helps when capturing canvas.

* Apply formatting changes

* Include all test image snapshots in ci

* Apply formatting changes

* Allow more flexibility when capturing hover

* Apply formatting changes

* Create tiny-chairs-build.md

* Apply formatting changes

* Update hover.test.ts

* Apply formatting changes

* Document snapshotFormat jest config

* Freeze `yarn.lock` in ci for reproducible dependencies

* Apply formatting changes

* Apply formatting changes

* Revert to old style of puppeteer evaluation script notation

* Apply formatting changes

* Make test less flaky

* Apply formatting changes

* Apply formatting changes

* Make tests less flaky

* Apply formatting changes

* Make test more robust

* Apply formatting changes

* Apply formatting changes

* Add debugging code for test

* Apply formatting changes

* Also test not ignored input

* Apply formatting changes

* Apply formatting changes

* Apply formatting changes

* escape ignoreSelector

* Apply formatting changes

* Apply formatting changes

Author: Juice10

.changeset/tiny-chairs-build.md

@@ -0,0 +1,7 @@
+---
+'rrweb': patch
+---
+
+Canvas FPS recording: override `preserveDrawingBuffer: true` on canvas creation.
+Canvas replay: fix flickering canvas elemenrs.
+Canvas FPS recording: fix bug that wipes webgl(2) canvas backgrounds while recording.

.github/workflows/ci-cd.yml

@@ -21,7 +21,7 @@ jobs:
           node-version: lts/*
 
       - name: Install Dependencies
-        run: yarn
+        run: yarn install --frozen-lockfile
 
       - name: Build Project
         run: NODE_OPTIONS='--max-old-space-size=4096' yarn build:all
@@ -37,5 +37,5 @@ jobs:
         if: failure()
         with:
           name: image-diff
-          path: packages/rrweb/test/e2e/__image_snapshots__/__diff_output__/*.png
+          path: packages/rrweb/test/*/__image_snapshots__/__diff_output__/*.png
           if-no-files-found: ignore

.github/workflows/release.yml

@@ -21,15 +21,15 @@ jobs:
           node-version: lts/*
 
       - name: Install Dependencies
-        run: yarn
+        run: yarn install --frozen-lockfile
 
       - name: Create Release Pull Request or Publish to npm
         id: changesets
         uses: changesets/action@v1
         with:
           publish: yarn run release
         env:
-          NODE_OPTIONS: "--max-old-space-size=4096"
+          NODE_OPTIONS: '--max-old-space-size=4096'
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
 

.github/workflows/style-check.yml

@@ -18,7 +18,7 @@ jobs:
           node-version: 16
           cache: 'yarn'
       - name: Install Dependencies
-        run: yarn
+        run: yarn install --frozen-lockfile
       - name: Build Packages
         run: NODE_OPTIONS='--max-old-space-size=4096' yarn build:all
       - name: Eslint Check

packages/rrweb/jest.config.js

@@ -6,4 +6,13 @@ export default {
   moduleNameMapper: {
     '\\.css$': 'identity-obj-proxy',
   },
+  /**
+   * Keeps old (pre-jest 29) snapshot format
+   * its a bit ugly and harder to read than the new format,
+   * so we might want to remove this in its own PR
+   */
+  snapshotFormat: {
+    escapeString: true,
+    printBasicPrototype: true,
+  },
 };

packages/rrweb/package.json

@@ -53,30 +53,30 @@
     "@types/chai": "^4.1.6",
     "@types/dom-mediacapture-transform": "^0.1.3",
     "@types/inquirer": "^8.2.1",
-    "@types/jest": "^27.4.1",
-    "@types/jest-image-snapshot": "^5.1.0",
+    "@types/jest": "^29.5.0",
+    "@types/jest-image-snapshot": "^6.1.0",
     "@types/node": "^18.15.11",
     "@types/offscreencanvas": "^2019.6.4",
-    "@types/puppeteer": "^5.4.4",
     "construct-style-sheets-polyfill": "^3.1.0",
     "cross-env": "^5.2.0",
     "esbuild": "^0.14.38",
     "fast-mhtml": "^1.1.9",
     "identity-obj-proxy": "^3.0.0",
     "ignore-styles": "^5.0.1",
     "inquirer": "^9.0.0",
-    "jest": "^27.5.1",
-    "jest-image-snapshot": "^5.2.0",
-    "jest-snapshot": "^23.6.0",
-    "puppeteer": "^11.0.0",
+    "jest": "^29.6.0",
+    "jest-environment-jsdom": "^29.6.0",
+    "jest-image-snapshot": "^6.2.0",
+    "jest-snapshot": "^29.6.2",
+    "puppeteer": "^20.9.0",
     "rollup": "^2.68.0",
     "rollup-plugin-esbuild": "^4.9.1",
     "rollup-plugin-postcss": "^3.1.1",
     "rollup-plugin-rename-node-modules": "^1.3.1",
     "rollup-plugin-typescript2": "^0.31.2",
     "rollup-plugin-web-worker-loader": "^1.6.1",
     "simple-peer-light": "^9.10.0",
-    "ts-jest": "^27.1.3",
+    "ts-jest": "^29.1.1",
     "ts-node": "^10.9.1",
     "tslib": "^2.3.1"
   },

packages/rrweb/src/record/observers/canvas/canvas-manager.ts

@@ -114,6 +114,7 @@ export class CanvasManager {
       win,
       blockClass,
       blockSelector,
+      true,
     );
     const snapshotInProgressMap: Map<number, boolean> = new Map();
     const worker =
@@ -198,9 +199,12 @@ export class CanvasManager {
             ) {
               // Hack to load canvas back into memory so `createImageBitmap` can grab it's contents.
               // Context: https://twitter.com/Juice10/status/1499775271758704643
-              // This hack might change the background color of the canvas in the unlikely event that
+              // Preferably we set `preserveDrawingBuffer` to true, but that's not always possible,
+              // especially when canvas is loaded before rrweb.
+              // This hack can wipe the background color of the canvas in the (unlikely) event that
               // the canvas background was changed but clear was not called directly afterwards.
-              context?.clear(context.COLOR_BUFFER_BIT);
+              // Example of this hack having negative side effect: https://visgl.github.io/react-map-gl/examples/layers
+              context.clear(context.COLOR_BUFFER_BIT);
             }
           }
           const bitmap = await createImageBitmap(canvas);
@@ -238,6 +242,7 @@ export class CanvasManager {
       win,
       blockClass,
       blockSelector,
+      false,
     );
     const canvas2DReset = initCanvas2DMutationObserver(
       this.processMutation.bind(this),

packages/rrweb/src/record/observers/canvas/canvas.ts

@@ -2,10 +2,15 @@ import type { ICanvas } from 'rrweb-snapshot';
 import type { blockClass, IWindow, listenerHandler } from '@rrweb/types';
 import { isBlocked, patch } from '../../../utils';
 
+function getNormalizedContextName(contextType: string) {
+  return contextType === 'experimental-webgl' ? 'webgl' : contextType;
+}
+
 export default function initCanvasContextObserver(
   win: IWindow,
   blockClass: blockClass,
   blockSelector: string | null,
+  setPreserveDrawingBufferToTrue: boolean,
 ): listenerHandler {
   const handlers: listenerHandler[] = [];
   try {
@@ -25,7 +30,24 @@ export default function initCanvasContextObserver(
           ...args: Array<unknown>
         ) {
           if (!isBlocked(this, blockClass, blockSelector, true)) {
-            if (!('__context' in this)) this.__context = contextType;
+            const ctxName = getNormalizedContextName(contextType);
+            if (!('__context' in this)) this.__context = ctxName;
+
+            if (
+              setPreserveDrawingBufferToTrue &&
+              ['webgl', 'webgl2'].includes(ctxName)
+            ) {
+              if (args[0] && typeof args[0] === 'object') {
+                const contextAttributes = args[0] as WebGLContextAttributes;
+                if (!contextAttributes.preserveDrawingBuffer) {
+                  contextAttributes.preserveDrawingBuffer = true;
+                }
+              } else {
+                args.splice(0, 1, {
+                  preserveDrawingBuffer: true,
+                });
+              }
+            }
           }
           return original.apply(this, [contextType, ...args]);
         };

packages/rrweb/src/replay/canvas/2d.ts

@@ -4,48 +4,63 @@ import { deserializeArg } from './deserialize-args';
 
 export default async function canvasMutation({
   event,
-  mutation,
+  mutations,
   target,
   imageMap,
   errorHandler,
 }: {
   event: Parameters<Replayer['applyIncremental']>[0];
-  mutation: canvasMutationCommand;
+  mutations: canvasMutationCommand[];
   target: HTMLCanvasElement;
   imageMap: Replayer['imageMap'];
   errorHandler: Replayer['warnCanvasMutationFailed'];
 }): Promise<void> {
-  try {
-    const ctx = target.getContext('2d')!;
+  const ctx = target.getContext('2d');
 
-    if (mutation.setter) {
-      // skip some read-only type checks
-      (ctx as unknown as Record<string, unknown>)[mutation.property] =
-        mutation.args[0];
-      return;
-    }
-    const original = ctx[
-      mutation.property as Exclude<keyof typeof ctx, 'canvas'>
-    ] as (ctx: CanvasRenderingContext2D, args: unknown[]) => void;
+  if (!ctx) {
+    errorHandler(mutations[0], new Error('Canvas context is null'));
+    return;
+  }
+
+  // step 1, deserialize args, they may be async
+  const mutationArgsPromises = mutations.map(
+    async (mutation: canvasMutationCommand): Promise<unknown[]> => {
+      return Promise.all(mutation.args.map(deserializeArg(imageMap, ctx)));
+    },
+  );
+  const args = await Promise.all(mutationArgsPromises);
+  // step 2 apply all mutations
+  args.forEach((args, index) => {
+    const mutation = mutations[index];
+    try {
+      if (mutation.setter) {
+        // skip some read-only type checks
+        (ctx as unknown as Record<string, unknown>)[mutation.property] =
+          mutation.args[0];
+        return;
+      }
+      const original = ctx[
+        mutation.property as Exclude<keyof typeof ctx, 'canvas'>
+      ] as (ctx: CanvasRenderingContext2D, args: unknown[]) => void;
 
-    /**
-     * We have serialized the image source into base64 string during recording,
-     * which has been preloaded before replay.
-     * So we can get call drawImage SYNCHRONOUSLY which avoid some fragile cast.
-     */
-    if (
-      mutation.property === 'drawImage' &&
-      typeof mutation.args[0] === 'string'
-    ) {
-      imageMap.get(event);
-      original.apply(ctx, mutation.args);
-    } else {
-      const args = await Promise.all(
-        mutation.args.map(deserializeArg(imageMap, ctx)),
-      );
-      original.apply(ctx, args);
+      /**
+       * We have serialized the image source into base64 string during recording,
+       * which has been preloaded before replay.
+       * So we can get call drawImage SYNCHRONOUSLY which avoid some fragile cast.
+       */
+      if (
+        mutation.property === 'drawImage' &&
+        typeof mutation.args[0] === 'string'
+      ) {
+        imageMap.get(event);
+        original.apply(ctx, mutation.args);
+      } else {
+        original.apply(ctx, args);
+      }
+    } catch (error) {
+      errorHandler(mutation, error);
     }
-  } catch (error) {
-    errorHandler(mutation, error);
-  }
+
+    return;
+  });
 }

packages/rrweb/src/replay/canvas/index.ts

@@ -46,16 +46,13 @@ export default async function canvasMutation({
       return;
     }
     // default is '2d' for backwards compatibility (rrweb below 1.1.x)
-    for (let i = 0; i < commands.length; i++) {
-      const command = commands[i];
-      await canvas2DMutation({
-        event,
-        mutation: command,
-        target,
-        imageMap,
-        errorHandler,
-      });
-    }
+    await canvas2DMutation({
+      event,
+      mutations: commands,
+      target,
+      imageMap,
+      errorHandler,
+    });
   } catch (error) {
     errorHandler(mutation, error);
   }