Setup basic WebRTC canvas streaming

packages/rrweb/package.json

@@ -46,6 +46,7 @@
   "devDependencies": {
     "@rollup/plugin-node-resolve": "^13.1.3",
     "@types/chai": "^4.1.6",
+    "@types/dom-mediacapture-transform": "^0.1.3",
     "@types/inquirer": "^8.2.1",
     "@types/jest": "^27.4.1",
     "@types/jest-image-snapshot": "^4.3.1",
@@ -70,6 +71,7 @@
     "rollup-plugin-rename-node-modules": "^1.3.1",
     "rollup-plugin-typescript2": "^0.31.2",
     "rollup-plugin-web-worker-loader": "^1.6.1",
+    "simple-peer-light": "^9.10.0",
     "ts-jest": "^27.1.3",
     "ts-node": "^10.9.1",
     "tslib": "^2.3.1",

packages/rrweb/scripts/stream.js

@@ -9,6 +9,10 @@ import { fileURLToPath } from 'url';
 
 import { startServer, getServerURL } from './utils.js';
 
+// Turn on devtools for debugging:
+const devtools = false;
+const defaultURL = 'https://webglsamples.org/';
+
 const __filename = fileURLToPath(import.meta.url);
 const __dirname = path.dirname(__filename);
 
@@ -20,6 +24,8 @@ function getCode() {
 }
 
 async function startRecording(page, serverURL) {
+  page.on('recording - console', (msg) => console.log('PAGE LOG:', msg.text()));
+
   await page.evaluate((serverURL) => {
     const el = document.createElement('script');
     el.addEventListener('load', () => {
@@ -34,14 +40,25 @@ async function startRecording(page, serverURL) {
         recordCanvas: true,
         collectFonts: true,
         inlineImages: true,
+        sampling: {
+          canvas: 'webrtc',
+        },
       });
     });
     el.src = `${serverURL}/rrweb.js`;
     document.head.appendChild(el);
   }, serverURL);
 }
 
-function startReplay(page, serverURL) {
+async function startReplay(page, serverURL, recordedPage) {
+  page.on('replay - console', (msg) => console.log('PAGE LOG:', msg.text()));
+
+  await page.exposeFunction('_signal', (signal) => {
+    recordedPage.evaluate((signal) => {
+      window.p.signal(signal);
+    }, signal);
+  });
+
   return page.evaluate((serverURL) => {
     const el = document.createElement('script');
     el.addEventListener('load', () => {
@@ -68,9 +85,8 @@ async function resizeWindow(page, top, left, width, height) {
 
 void (async () => {
   const code = getCode();
-  const server = await startServer();
-  const serverURL = getServerURL(server);
-  let events = [];
+  let server;
+  let serverURL;
 
   await start();
 
@@ -90,13 +106,14 @@ void (async () => {
   };
 
   async function start() {
-    events = [];
+    server = await startServer();
+    serverURL = getServerURL(server);
+
     const { url } = await inquirer.prompt([
       {
         type: 'input',
         name: 'url',
-        message:
-          'Enter the url you want to record, e.g https://react-redux.realworld.io: ',
+        message: `Enter the url you want to record, e.g ${defaultURL}: `,
       },
     ]);
 
@@ -120,11 +137,11 @@ void (async () => {
   }
 
   async function record(url) {
-    if (url === '') url = 'https://react-redux.realworld.io';
+    if (url === '') url = defaultURL;
 
     const replayingBrowser = await puppeteer.launch({
       headless: false,
-      // devtools: true,
+      devtools,
       defaultViewport: {
         width: 1600,
         height: 900,
@@ -142,11 +159,10 @@ void (async () => {
     await replayerPage.addStyleTag({
       path: path.resolve(__dirname, '../dist/rrweb.min.css'),
     });
-    await startReplay(replayerPage, serverURL);
 
     const recordingBrowser = await puppeteer.launch({
       headless: false,
-      // devtools: true,
+      devtools,
       defaultViewport: {
         width: 1600,
         height: 900,
@@ -165,6 +181,8 @@ void (async () => {
     // disables content security policy which enables us to insert rrweb as a script tag
     await recordedPage.setBypassCSP(true);
 
+    await startReplay(replayerPage, serverURL, recordedPage);
+
     await Promise.all([
       resizeWindow(recordedPage, 0, 0, 800, 800),
       resizeWindow(replayerPage, 0, 800, 800, 800),
@@ -178,7 +196,9 @@ void (async () => {
     if (!replayerPage) throw new Error('No replayer page found');
 
     await recordedPage.exposeFunction('_replLog', (event) => {
-      replayerPage.evaluate((event) => window.replayer?.addEvent(event), event);
+      replayerPage.evaluate((event) => {
+        window.replayer?.addEvent(event);
+      }, event);
     });
     await startRecording(recordedPage, serverURL);
     recordedPage.on('framenavigated', async () => {

packages/rrweb/src/record/observers/canvas/canvas-manager.ts

@@ -15,6 +15,7 @@ import initCanvasContextObserver from './canvas';
 import initCanvasWebGLMutationObserver from './webgl';
 import ImageBitmapDataURLWorker from 'web-worker:../../workers/image-bitmap-data-url-worker.ts';
 import type { ImageBitmapDataURLRequestWorker } from '../../workers/image-bitmap-data-url-worker';
+import SimplePeer from 'simple-peer-light';
 
 export type RafStamps = { latestId: number; invokeId: number | null };
 
@@ -60,12 +61,14 @@ export class CanvasManager {
     win: IWindow;
     blockClass: blockClass;
     mirror: Mirror;
-    sampling?: 'all' | number;
+    sampling?: 'all' | 'webrtc' | number;
   }) {
     const { sampling = 'all', win, blockClass, recordCanvas } = options;
     this.mutationCb = options.mutationCb;
     this.mirror = options.mirror;
 
+    if (recordCanvas && sampling === 'webrtc')
+      this.initCanvasWebRTCObserver(win, blockClass);
     if (recordCanvas && sampling === 'all')
       this.initCanvasMutationObserver(win, blockClass);
     if (recordCanvas && typeof sampling === 'number')
@@ -195,6 +198,64 @@ export class CanvasManager {
     };
   }
 
+  // private sdpOffer: RTCSessionDescriptionInit;
+
+  private setupPeer(id: number, stream: MediaStream) {
+    console.log('setupPeer', id);
+    const p = new SimplePeer({
+      initiator: true,
+      stream,
+      trickle: false, // only send one WebRTC offer per canvas
+    });
+
+    (window as any).p = p;
+
+    p.on('signal', (data: RTCSessionDescriptionInit) => {
+      this.mutationCb({
+        id,
+        type: CanvasContext.WebRTC,
+        msg: data,
+        stream,
+      });
+    });
+
+    p.on('connect', () => {
+      // eslint-disable-next-line prefer-rest-params
+      console.log('connected', arguments);
+    });
+  }
+
+  private initCanvasWebRTCObserver(win: IWindow, blockClass: blockClass) {
+    const canvasContextReset = initCanvasContextObserver(win, blockClass);
+    const streamMap: Map<number, MediaStream> = new Map();
+    let rafId: number;
+
+    const captureStreams = () => {
+      win.document
+        .querySelectorAll(`canvas:not(.${blockClass as string} *)`)
+        .forEach((canvas: HTMLCanvasElement) => {
+          const id = this.mirror.getId(canvas);
+          if (streamMap.has(id)) return;
+
+          // streamMap.set(id, (null as unknown) as MediaStream);
+          // setTimeout(() => {
+          console.log('captureStream', canvas, canvas.width, canvas.height);
+          const stream = canvas.captureStream();
+          streamMap.set(id, stream);
+          this.setupPeer(id, stream);
+          // }, 2000);
+        });
+      rafId = requestAnimationFrame(captureStreams);
+    };
+
+    rafId = requestAnimationFrame(captureStreams);
+
+    this.resetObservers = () => {
+      canvasContextReset();
+      cancelAnimationFrame(rafId);
+    };
+  }
+
   private initCanvasMutationObserver(
     win: IWindow,
     blockClass: blockClass,