⚗️ Capture dirty canvas images

packages/browser-rum/src/domain/record/canvas/canvasCapture.spec.ts

@@ -0,0 +1,149 @@
+import { collectAsyncCalls, mockClock, registerCleanupTask } from '@datadog/browser-core/test'
+import type { Clock } from '@datadog/browser-core/test'
+import { ONE_SECOND } from '@datadog/js-core/time'
+import type { Tracker } from '../trackers'
+import { createCanvasManager } from './canvasManager'
+import type { ComputeCanvasImageHash, EmitCanvasImage } from './canvasCapture'
+import { computeCanvasImageHash, startCanvasCapture } from './canvasCapture'
+
+describe('startCanvasCapture', () => {
+  let canvas: HTMLCanvasElement
+  let canvasManager: ReturnType<typeof createCanvasManager>
+  let clock: Clock
+  let computeImageHashSpy: jasmine.Spy<ComputeCanvasImageHash>
+  let emitCanvasImageSpy: jasmine.Spy<EmitCanvasImage>
+  let imageBlob: Blob
+  let toBlobSpy: jasmine.Spy<HTMLCanvasElement['toBlob']>
+  let tracker: Tracker | undefined
+
+  beforeEach(() => {
+    canvas = document.createElement('canvas')
+    canvasManager = createCanvasManager()
+    clock = mockClock()
+    imageBlob = new Blob(['frame'], { type: 'image/png' })
+    computeImageHashSpy = jasmine.createSpy().and.resolveTo('frame-hash')
+    emitCanvasImageSpy = jasmine.createSpy()
+    toBlobSpy = spyOn(canvas, 'toBlob').and.callFake((callback) => callback(imageBlob))
+
+    registerCleanupTask(() => tracker?.stop())
+  })
+
+  it('captures dirty canvases at the configured maximum frame rate', async () => {
+    tracker = startCanvasCapture(canvasManager, 2, emitCanvasImageSpy, computeImageHashSpy)
+    canvasManager.markCanvasDirty(canvas)
+
+    clock.tick(499)
+    expect(toBlobSpy).not.toHaveBeenCalled()
+
+    clock.tick(1)
+    await collectAsyncCalls(emitCanvasImageSpy)
+
+    expect(toBlobSpy).toHaveBeenCalledOnceWith(jasmine.any(Function), 'image/png')
+    expect(emitCanvasImageSpy).toHaveBeenCalledOnceWith({
+      blob: imageBlob,
+      canvas,
+      hash: 'frame-hash',
+    })
+    expect(canvasManager.isCanvasDirty(canvas)).toBeFalse()
+  })
+
+  it('does not capture clean canvases', () => {
+    tracker = startCanvasCapture(canvasManager, 1, emitCanvasImageSpy, computeImageHashSpy)
+
+    clock.tick(ONE_SECOND)
+
+    expect(toBlobSpy).not.toHaveBeenCalled()
+  })
+
+  it('skips a captured image when its hash has not changed', async () => {
+    tracker = startCanvasCapture(canvasManager, 1, emitCanvasImageSpy, computeImageHashSpy)
+
+    canvasManager.markCanvasDirty(canvas)
+    clock.tick(ONE_SECOND)
+    await collectAsyncCalls(emitCanvasImageSpy)
+
+    canvasManager.markCanvasDirty(canvas)
+    const secondHash = collectAsyncCalls(computeImageHashSpy, 2)
+    clock.tick(ONE_SECOND)
+    await secondHash
+    await Promise.resolve()
+
+    expect(toBlobSpy).toHaveBeenCalledTimes(2)
+    expect(emitCanvasImageSpy).toHaveBeenCalledTimes(1)
+  })
+
+  it('emits each transition when an image returns to an earlier hash', async () => {
+    computeImageHashSpy.and.resolveTo('first-hash')
+    tracker = startCanvasCapture(canvasManager, 1, emitCanvasImageSpy, computeImageHashSpy)
+
+    canvasManager.markCanvasDirty(canvas)
+    clock.tick(ONE_SECOND)
+    await collectAsyncCalls(emitCanvasImageSpy)
+
+    computeImageHashSpy.and.resolveTo('second-hash')
+    canvasManager.markCanvasDirty(canvas)
+    const secondImage = collectAsyncCalls(emitCanvasImageSpy, 2)
+    clock.tick(ONE_SECOND)
+    await secondImage
+
+    computeImageHashSpy.and.resolveTo('first-hash')
+    canvasManager.markCanvasDirty(canvas)
+    const thirdImage = collectAsyncCalls(emitCanvasImageSpy, 3)
+    clock.tick(ONE_SECOND)
+    await thirdImage
+
+    expect(emitCanvasImageSpy.calls.argsFor(1)[0].hash).toBe('second-hash')
+    expect(emitCanvasImageSpy.calls.argsFor(2)[0].hash).toBe('first-hash')
+  })
+
+  it('keeps changes made while an image capture is in progress dirty', async () => {
+    let resolveHash!: (hash: string) => void
+    computeImageHashSpy.and.returnValue(new Promise((resolve) => (resolveHash = resolve)))
+    tracker = startCanvasCapture(canvasManager, 1, emitCanvasImageSpy, computeImageHashSpy)
+
+    canvasManager.markCanvasDirty(canvas)
+    clock.tick(ONE_SECOND)
+    canvasManager.markCanvasDirty(canvas)
+    clock.tick(ONE_SECOND)
+
+    expect(toBlobSpy).toHaveBeenCalledTimes(1)
+    expect(canvasManager.isCanvasDirty(canvas)).toBeTrue()
+
+    resolveHash('first-hash')
+    await collectAsyncCalls(emitCanvasImageSpy)
+    await Promise.resolve()
+    clock.tick(ONE_SECOND)
+
+    expect(toBlobSpy).toHaveBeenCalledTimes(2)
+  })
+
+  it('ignores captures that do not produce a blob', () => {
+    toBlobSpy.and.callFake((callback) => callback(null))
+    tracker = startCanvasCapture(canvasManager, 1, emitCanvasImageSpy, computeImageHashSpy)
+    canvasManager.markCanvasDirty(canvas)
+
+    clock.tick(ONE_SECOND)
+
+    expect(computeImageHashSpy).not.toHaveBeenCalled()
+    expect(emitCanvasImageSpy).not.toHaveBeenCalled()
+  })
+
+  it('stops capturing images and clears dirty canvases', () => {
+    tracker = startCanvasCapture(canvasManager, 1, emitCanvasImageSpy, computeImageHashSpy)
+    canvasManager.markCanvasDirty(canvas)
+
+    tracker.stop()
+    clock.tick(ONE_SECOND)
+
+    expect(toBlobSpy).not.toHaveBeenCalled()
+    expect(canvasManager.isCanvasDirty(canvas)).toBeFalse()
+  })
+})
+
+describe('computeCanvasImageHash', () => {
+  it('computes a SHA-256 hash from the image bytes', async () => {
+    expect(await computeCanvasImageHash(new Blob(['hello']))).toBe(
+      '2cf24dba5fb0a30e26e83b2ac5b9e29e1b161e5c1fa7425e73043362938b9824'
+    )
+  })
+})

packages/browser-rum/src/domain/record/canvas/canvasCapture.ts

@@ -0,0 +1,89 @@
+import { clearInterval, setInterval } from '@datadog/browser-core'
+import { ONE_SECOND } from '@datadog/js-core/time'
+import type { Tracker } from '../trackers'
+import type { CanvasManager } from './canvasManager'
+
+export interface CapturedCanvasImage {
+  blob: Blob
+  canvas: HTMLCanvasElement
+  hash: string
+}
+
+export type ComputeCanvasImageHash = (blob: Blob) => Promise<string>
+export type EmitCanvasImage = (image: CapturedCanvasImage) => void
+
+export function startCanvasCapture(
+  canvasManager: CanvasManager,
+  maxFramesPerSecond: number,
+  emitCanvasImage: EmitCanvasImage,
+  computeImageHash: ComputeCanvasImageHash = computeCanvasImageHash
+): Tracker {
+  const canvasesBeingCaptured = new WeakSet<HTMLCanvasElement>()
+  const lastCanvasHash = new WeakMap<HTMLCanvasElement, string>()
+  let stopped = false
+
+  const captureIntervalId = setInterval(captureDirtyCanvases, ONE_SECOND / maxFramesPerSecond)
+
+  function captureDirtyCanvases() {
+    canvasManager.getDirtyCanvases().forEach((canvas) => {
+      if (canvasesBeingCaptured.has(canvas)) {
+        return
+      }
+
+      canvasManager.markCanvasClean(canvas)
+      canvasesBeingCaptured.add(canvas)
+
+      try {
+        canvas.toBlob((blob) => {
+          if (stopped || !blob) {
+            canvasesBeingCaptured.delete(canvas)
+            return
+          }
+
+          void computeImageHash(blob)
+            .then((hash) => {
+              canvasesBeingCaptured.delete(canvas)
+
+              if (stopped || lastCanvasHash.get(canvas) === hash) {
+                return
+              }
+
+              lastCanvasHash.set(canvas, hash)
+              emitCanvasImage({ blob, canvas, hash })
+            })
+            .catch(() => canvasesBeingCaptured.delete(canvas))
+        }, 'image/png')
+      } catch {
+        canvasesBeingCaptured.delete(canvas)
+      }
+    })
+  }
+
+  return {
+    stop: () => {
+      stopped = true
+      clearInterval(captureIntervalId)
+      canvasManager.clearDirtyCanvases()
+    },
+  }
+}
+
+export async function computeCanvasImageHash(blob: Blob): Promise<string> {
+  const imageBytes = await readBlobAsArrayBuffer(blob)
+  const digest = await crypto.subtle.digest('SHA-256', imageBytes)
+
+  return Array.from(new Uint8Array(digest), (byte) => byte.toString(16).padStart(2, '0')).join('')
+}
+
+function readBlobAsArrayBuffer(blob: Blob): Promise<ArrayBuffer> {
+  if (typeof blob.arrayBuffer === 'function') {
+    return blob.arrayBuffer()
+  }
+
+  return new Promise((resolve, reject) => {
+    const reader = new FileReader()
+    reader.onload = () => resolve(reader.result as ArrayBuffer)
+    reader.onerror = () => reject(reader.error || new Error('Unable to read canvas image'))
+    reader.readAsArrayBuffer(blob)
+  })
+}

packages/browser-rum/src/domain/record/canvas/canvasManager.spec.ts

@@ -23,5 +23,18 @@ describe('createCanvasManager', () => {
 
     expect(canvasManager.isCanvasDirty(dirtyCanvas)).toBeTrue()
     expect(canvasManager.isCanvasDirty(cleanCanvas)).toBeFalse()
+    expect(canvasManager.getDirtyCanvases()).toEqual([dirtyCanvas])
+  })
+
+  it('clears all dirty canvases', () => {
+    const canvasManager = createCanvasManager()
+    const firstCanvas = document.createElement('canvas')
+    const secondCanvas = document.createElement('canvas')
+
+    canvasManager.markCanvasDirty(firstCanvas)
+    canvasManager.markCanvasDirty(secondCanvas)
+    canvasManager.clearDirtyCanvases()
+
+    expect(canvasManager.getDirtyCanvases()).toEqual([])
   })
 })

packages/browser-rum/src/domain/record/canvas/canvasManager.ts

@@ -1,13 +1,17 @@
 export interface CanvasManager {
+  clearDirtyCanvases: () => void
+  getDirtyCanvases: () => HTMLCanvasElement[]
   isCanvasDirty: (canvas: HTMLCanvasElement) => boolean
   markCanvasClean: (canvas: HTMLCanvasElement) => void
   markCanvasDirty: (canvas: HTMLCanvasElement) => void
 }
 
 export function createCanvasManager(): CanvasManager {
-  const dirtyCanvases = new WeakSet<HTMLCanvasElement>()
+  const dirtyCanvases = new Set<HTMLCanvasElement>()
 
   return {
+    clearDirtyCanvases: () => dirtyCanvases.clear(),
+    getDirtyCanvases: () => Array.from(dirtyCanvases),
     isCanvasDirty: (canvas) => dirtyCanvases.has(canvas),
     markCanvasClean: (canvas) => dirtyCanvases.delete(canvas),
     markCanvasDirty: (canvas) => dirtyCanvases.add(canvas),

packages/browser-rum/src/domain/record/record.spec.ts

@@ -103,6 +103,31 @@ describe('record', () => {
         originalFillRect
       )
     })
+
+    it('captures dirty canvases at the configured maximum frame rate', async () => {
+      const clock = mockClock()
+      const canvas = document.createElement('canvas')
+      const context = canvas.getContext('2d')!
+      const blob = new Blob(['frame'], { type: 'image/png' })
+      const capturedImageSpy = jasmine.createSpy()
+      spyOn(canvas, 'toBlob').and.callFake((callback) => callback(blob))
+
+      startRecording({ recordCanvas: true, canvasMaxFramesPerSecond: 1 })
+      recordApi.canvasImageObservable.subscribe(capturedImageSpy)
+      context.fillRect(0, 0, 1, 1)
+      clock.tick(999)
+
+      expect(capturedImageSpy).not.toHaveBeenCalled()
+
+      clock.tick(1)
+      await collectAsyncCalls(capturedImageSpy)
+
+      expect(capturedImageSpy).toHaveBeenCalledOnceWith({
+        blob,
+        canvas,
+        hash: jasmine.any(String),
+      })
+    })
   })
 
   it('flushes pending mutation records before taking a full snapshot', async () => {

packages/browser-rum/src/domain/record/record.ts

@@ -1,4 +1,4 @@
-import { sendToExtension } from '@datadog/browser-core'
+import { Observable, sendToExtension } from '@datadog/browser-core'
 import type { LifeCycle, RumConfiguration, ViewHistory } from '@datadog/browser-rum-core'
 import * as replayStats from '../replayStats'
 import type { BrowserRecord } from '../../types'
@@ -24,6 +24,8 @@ import { startFullSnapshots } from './startFullSnapshots'
 import type { EmitRecordCallback, EmitStatsCallback } from './record.types'
 import { createRecordingScope } from './recordingScope'
 import { createCanvasManager } from './canvas/canvasManager'
+import type { CapturedCanvasImage } from './canvas/canvasCapture'
+import { startCanvasCapture } from './canvas/canvasCapture'
 
 export interface RecordOptions {
   emitRecord: EmitRecordCallback
@@ -34,13 +36,15 @@ export interface RecordOptions {
 }
 
 export interface RecordAPI {
+  canvasImageObservable: Observable<CapturedCanvasImage>
   stop: () => void
   flushMutations: () => void
   shadowRootsController: ShadowRootsController
 }
 
 export function record(options: RecordOptions): RecordAPI {
   const { emitRecord, emitStats, configuration, lifeCycle } = options
+  const canvasImageObservable = new Observable<CapturedCanvasImage>()
   // runtime checks for user options
   if (!emitRecord || !emitStats) {
     throw new Error('emit functions are required')
@@ -83,10 +87,16 @@ export function record(options: RecordOptions): RecordAPI {
     configuration.enableSessionReplayCanvasRecording.maxFramesPerSecond > 0
   ) {
     const canvasManager = createCanvasManager()
-    trackers.push(trackCanvas2DMutations(canvasManager.markCanvasDirty))
+    trackers.push(
+      trackCanvas2DMutations(canvasManager.markCanvasDirty),
+      startCanvasCapture(canvasManager, configuration.canvasMaxFramesPerSecond, (image) =>
+        canvasImageObservable.notify(image)
+      )
+    )
   }
 
   return {
+    canvasImageObservable,
     stop: () => {
       shadowRootsController.stop()
       trackers.forEach((tracker) => tracker.stop())