Merge pull request #5 from kaltura/fix/issue-3-stall-detection

fix: detect and signal stalled frame delivery via a watchdog

Author: zoharbabin

src/chromakey.js

@@ -90,6 +90,13 @@ export const DEFAULTS = Object.freeze({
   /** Pixel budget for the CPU fallback; frames are processed downscaled to fit. */
   maxCPUPixels: 512 * 512,
 
+  /**
+   * Milliseconds of no decoded-frame progress (while playing) before firing
+   * `'stalled'` — catches a track that goes silent while readyState/paused/
+   * ended still look healthy (e.g. a WebRTC failure). `<= 0` disables it.
+   */
+  stallTimeout: 4000,
+
   /** Attributes applied to the <video> element created when `source` is a URL. */
   videoAttributes: Object.freeze({
     muted: true,
@@ -103,6 +110,9 @@ export const DEFAULTS = Object.freeze({
 /** Options that select shader variants / backends / plugins and can't change after construction. */
 const IMMUTABLE_OPTIONS = ['channel', 'forceCanvas2D', 'videoAttributes', 'autoTune'];
 
+/** Watchdog polling cadence for stall detection (see `_checkStall`). Fixed; `stallTimeout` is the tunable. */
+const STALL_CHECK_INTERVAL_MS = 500;
+
 function clamp(x, lo, hi) { return x < lo ? lo : x > hi ? hi : x; }
 
 /** GLSL-compatible smoothstep, used by the CPU fallback fade ramps. */
@@ -631,6 +641,12 @@ function syncAspectRatio(target, video, state) {
  * - `'autotune'` — detail: the result object of every {@link autoTune} run.
  * - `'pluginerror'` — detail: `{ plugin, error }`, fired when a plugin hook
  *   throws (the plugin is detached; the player keeps running).
+ * - `'stalled'` — detail: `{ elapsedMs }`. Fired once when no new decoded
+ *   frame has arrived for `options.stallTimeout` ms while playing (see
+ *   issue #3). The canvas keeps showing the last good frame; the page
+ *   decides what to do (spinner, reconnect, etc).
+ * - `'recovered'` — detail: `{ elapsedMs }`. Fired once frame delivery
+ *   resumes after a `'stalled'` event.
  */
 export class ChromaKeyVideo extends EventTarget {
   /**
@@ -700,6 +716,11 @@ export class ChromaKeyVideo extends EventTarget {
 
     this._startLoop();
 
+    this._lastFrameQualityCount = -1;
+    this._stallStartedAt = 0;
+    this._isStalled = false;
+    this._stallWatchdog = setInterval(() => this._checkStall(), STALL_CHECK_INTERVAL_MS);
+
     if (this.options.autoTune) {
       this.use(autoTunePlugin({ adaptive: this.options.autoTune === 'adaptive' }));
     }
@@ -928,6 +949,7 @@ export class ChromaKeyVideo extends EventTarget {
       this.video.cancelVideoFrameCallback(this._rvfcHandle);
     }
     if (this._rafHandle) cancelAnimationFrame(this._rafHandle);
+    if (this._stallWatchdog) clearInterval(this._stallWatchdog);
     if (this._resizeObserver) this._resizeObserver.disconnect();
     for (const type of ['loadedmetadata', 'resize', 'loadeddata', 'seeked', 'error']) {
       this.video.removeEventListener(type, this._onVideoEvent);
@@ -1045,6 +1067,45 @@ export class ChromaKeyVideo extends EventTarget {
     syncAspectRatio(this.canvas, this.video, this);
   }
 
+  /**
+   * Watchdog tick: fires `'stalled'`/`'recovered'` off decoded-frame progress
+   * (see `options.stallTimeout`). Uses `getVideoPlaybackQuality()` rather
+   * than `currentTime` — for a live MediaStream, `currentTime` keeps
+   * advancing on the stream's own clock even when no new frame is decoded,
+   * so it would miss exactly the stall this exists to catch. See issue #3.
+   */
+  _checkStall() {
+    const timeout = this.options.stallTimeout;
+    if (!timeout || timeout <= 0) return;
+    if (typeof this.video.getVideoPlaybackQuality !== 'function') return;
+    if (this.video.paused || this.video.ended) {
+      this._stallStartedAt = 0;
+      return;
+    }
+
+    const count = this.video.getVideoPlaybackQuality().totalVideoFrames;
+    if (count !== this._lastFrameQualityCount) {
+      this._lastFrameQualityCount = count;
+      this._stallStartedAt = 0;
+      if (this._isStalled) {
+        this._isStalled = false;
+        this.dispatchEvent(new CustomEvent('recovered', { detail: { elapsedMs: this._stalledElapsedMs } }));
+      }
+      return;
+    }
+
+    if (!this._stallStartedAt) {
+      this._stallStartedAt = Date.now();
+      return;
+    }
+    const elapsedMs = Date.now() - this._stallStartedAt;
+    if (elapsedMs >= timeout && !this._isStalled) {
+      this._isStalled = true;
+      this._stalledElapsedMs = elapsedMs;
+      this.dispatchEvent(new CustomEvent('stalled', { detail: { elapsedMs } }));
+    }
+  }
+
   /** Redundant DOM-level bottom fade (defense in depth alongside the shader fade). */
   _applyCssFade() {
     const wanted = this.options.edgeDissolve && this.options.cssFade && this.options.fadeBottom > 0;
@@ -1306,6 +1367,7 @@ const ELEMENT_OPTION_ATTRIBUTES = {
   'fade-top': ['fadeTop', Number],
   'fade-bottom': ['fadeBottom', Number],
   'max-pixel-ratio': ['maxPixelRatio', Number],
+  'stall-timeout': ['stallTimeout', Number],
 };
 
 /**
@@ -1314,7 +1376,7 @@ const ELEMENT_OPTION_ATTRIBUTES = {
  * Supported attributes: `src` (required), `autoplay`, `loop`, `muted`,
  * `channel`, `min-key`, `bias`, `softness`, `spill`, `edge-dissolve`,
  * `auto-tune` (empty = once, `"adaptive"` = continuous), `fade-top`,
- * `fade-bottom`, `max-pixel-ratio`.
+ * `fade-bottom`, `max-pixel-ratio`, `stall-timeout`.
  *
  * The underlying player is exposed as the element's `.player` property for
  * full programmatic control (events, `update()`, the video element, etc.).

test/e2e.spec.js

@@ -44,6 +44,26 @@ function backend(page, id) {
   return page.evaluate((pid) => window.players[pid].player.backend, id);
 }
 
+// Resolves with the detail of the next `type` event on player `id`, or
+// rejects if it doesn't fire within `timeoutMs`.
+function waitForEvent(page, id, type, timeoutMs = 5000) {
+  return page.evaluate(([pid, t, ms]) => new Promise((resolve, reject) => {
+    const { player } = window.players[pid];
+    const timer = setTimeout(() => reject(new Error(`"${t}" never fired`)), ms);
+    player.addEventListener(t, (e) => { clearTimeout(timer); resolve(e.detail); }, { once: true });
+  }), [id, type, timeoutMs]);
+}
+
+// Resolves with whether `type` fired on player `id` within `waitMs`.
+function didFire(page, id, type, waitMs) {
+  return page.evaluate(([pid, t, ms]) => new Promise((resolve) => {
+    const { player } = window.players[pid];
+    let seen = false;
+    player.addEventListener(t, () => { seen = true; }, { once: true });
+    setTimeout(() => resolve(seen), ms);
+  }), [id, type, waitMs]);
+}
+
 test.describe('chroma-key-video', () => {
   test.beforeEach(async ({ page }) => {
     await openFixture(page);
@@ -450,4 +470,64 @@ test.describe('chroma-key-video', () => {
     expect(result.bg[3]).toBeLessThanOrEqual(30);
     expect(result.fg[3]).toBeGreaterThan(220);
   });
+
+  test('fires "stalled" when frame delivery stops, then "recovered" once it resumes (issue #3)', async ({ page }) => {
+    const id = await create(page, { stallTimeout: 700 }, { fps: 0 });
+
+    const healthBefore = await page.evaluate((pid) => {
+      const v = window.players[pid].player.video;
+      return { paused: v.paused, ended: v.ended };
+    }, id);
+    expect(healthBefore.paused).toBe(false);
+    expect(healthBefore.ended).toBe(false);
+
+    const stalledPromise = waitForEvent(page, id, 'stalled', 5000);
+    await page.evaluate((pid) => window.stopSource(pid), id);
+    const stalledDetail = await stalledPromise;
+    expect(stalledDetail.elapsedMs).toBeGreaterThanOrEqual(700);
+
+    // readyState/paused/ended still look healthy — that's the whole point:
+    // the video element gives no other signal that frames stopped arriving.
+    const healthDuringStall = await page.evaluate((pid) => {
+      const v = window.players[pid].player.video;
+      return { paused: v.paused, ended: v.ended };
+    }, id);
+    expect(healthDuringStall.paused).toBe(false);
+    expect(healthDuringStall.ended).toBe(false);
+
+    const recoveredPromise = waitForEvent(page, id, 'recovered', 5000);
+    await page.evaluate((pid) => window.resumeSource(pid), id);
+    await recoveredPromise;
+  });
+
+  test('does not fire "stalled" during uninterrupted playback (issue #3)', async ({ page }) => {
+    const id = await create(page, { stallTimeout: 300 }, { fps: 0 });
+    expect(await didFire(page, id, 'stalled', 1500)).toBe(false);
+  });
+
+  test('does not fire "stalled" while intentionally paused (issue #3)', async ({ page }) => {
+    const id = await create(page, { stallTimeout: 300 }, { fps: 0 });
+    await page.evaluate((pid) => {
+      window.players[pid].player.pause();
+      window.stopSource(pid);
+    }, id);
+    expect(await didFire(page, id, 'stalled', 1500)).toBe(false);
+  });
+
+  test('stallTimeout <= 0 disables stall detection (issue #3)', async ({ page }) => {
+    const id = await create(page, { stallTimeout: 0 }, { fps: 0 });
+    await page.evaluate((pid) => window.stopSource(pid), id);
+    expect(await didFire(page, id, 'stalled', 1500)).toBe(false);
+  });
+
+  test('stalling one instance does not affect another (issue #3)', async ({ page }) => {
+    const a = await create(page, { stallTimeout: 700 }, { fps: 0 });
+    const b = await create(page, { stallTimeout: 700 }, { fps: 0 });
+
+    const stalledA = waitForEvent(page, a, 'stalled', 5000);
+    await page.evaluate((pid) => window.stopSource(pid), a);
+    await stalledA;
+
+    expect(await didFire(page, b, 'stalled', 500)).toBe(false);
+  });
 });

test/fixture.html

@@ -43,24 +43,28 @@
 
 // captureStream only emits frames on draw commits, so keep redrawing the
 // (identical) pattern on an interval to feed the video a steady frame rate.
-function makePatternStream(channel = 'green', w = W, h = H) {
+// fps=0 uses the no-arg, draw-triggered captureStream() instead of a fixed
+// capture rate, so stopping the redraw interval genuinely stops new frames
+// from being produced (rather than the fixed-rate capture re-encoding the
+// same static content) — needed to simulate a real stalled track.
+function makePatternStream(channel = 'green', w = W, h = H, fps = 30) {
   const canvas = document.createElement('canvas');
   canvas.width = w;
   canvas.height = h;
   const ctx = canvas.getContext('2d');
   drawPattern(ctx, channel);
-  const stream = canvas.captureStream(30);
+  const stream = fps ? canvas.captureStream(fps) : canvas.captureStream();
   const timer = setInterval(() => drawPattern(ctx, channel), 50);
-  return { stream, canvas, timer };
+  return { stream, canvas, timer, ctx };
 }
 
 // Create a player from a synthetic pattern stream, mount it, wait for the
 // first rendered frame, and return its handle id. srcWidth/srcHeight set the
 // pattern canvas's (i.e. the video's native) resolution, independent of the
 // player canvas's CSS display size (cssWidth/cssHeight).
-window.createPlayer = (options = {}, { channel = 'green', cssWidth = W, cssHeight = H, srcWidth = W, srcHeight = H } = {}) => {
+window.createPlayer = (options = {}, { channel = 'green', cssWidth = W, cssHeight = H, srcWidth = W, srcHeight = H, fps = 30 } = {}) => {
   return new Promise((resolve, reject) => {
-    const { stream, canvas, timer } = makePatternStream(channel, srcWidth, srcHeight);
+    const { stream, canvas, timer, ctx } = makePatternStream(channel, srcWidth, srcHeight, fps);
     const player = new ChromaKeyVideo(stream, {
       maxPixelRatio: 1,
       videoAttributes: { ...DEFAULTS.videoAttributes, autoplay: true },
@@ -70,7 +74,7 @@
     player.canvas.style.height = cssHeight + 'px';
     player.mount(document.getElementById('stage'));
     const id = String(nextId++);
-    window.players[id] = { player, timer, sourceCanvas: canvas, channel };
+    window.players[id] = { player, timer, sourceCanvas: canvas, sourceCtx: ctx, channel };
     const timeout = setTimeout(() => reject(new Error('player never started')), 10000);
     player.addEventListener('started', () => {
       // Let a couple more frames land so sampling sees steady-state output.
@@ -81,6 +85,29 @@
   });
 };
 
+// Stops feeding new frames into a player's source stream, simulating a
+// track that goes silent while still reporting healthy paused/ended/readyState.
+window.stopSource = (id) => {
+  clearInterval(window.players[id].timer);
+  window.players[id].timer = 0;
+};
+
+// Resumes feeding frames after stopSource().
+window.resumeSource = (id) => {
+  const entry = window.players[id];
+  if (entry.timer) return;
+  entry.timer = setInterval(() => drawPattern(entry.sourceCtx, entry.channel), 50);
+};
+
+// Reports the video's decoded-frame counter, for diagnosing whether the
+// stall-detection signal (getVideoPlaybackQuality().totalVideoFrames) is
+// actually advancing/frozen in this browser/environment.
+window.playbackQuality = (id) => {
+  const v = window.players[id].player.video;
+  if (typeof v.getVideoPlaybackQuality !== 'function') return null;
+  return v.getVideoPlaybackQuality().totalVideoFrames;
+};
+
 // Resizes a player's source pattern canvas, simulating a live track's native
 // resolution changing mid-stream (e.g. a WebRTC renegotiation). The video
 // element fires its native 'resize' event once the new dimensions decode.