Merge pull request #5962 from remotion-dev/on-frame-callback

Author: JonnyBurger

packages/core/src/Artifact.tsx

@@ -1,5 +1,5 @@
 import type React from 'react';
-import {useContext, useEffect, useState} from 'react';
+import {useContext, useLayoutEffect, useState} from 'react';
 import {RenderAssetManager} from './RenderAssetManager';
 import type {DownloadBehavior} from './download-behavior';
 import {useCurrentFrame} from './use-current-frame';
@@ -25,7 +25,7 @@ export const Artifact: React.FC<{
 		return String(Math.random());
 	});
 
-	useEffect(() => {
+	useLayoutEffect(() => {
 		if (!env.isRendering) {
 			return;
 		}

packages/web-renderer/src/index.ts

@@ -14,3 +14,4 @@ export type {
 	RenderStillOnWebImageFormat as RenderStillImageFormat,
 	RenderStillOnWebOptions,
 } from './render-still-on-web';
+export type {OnFrameCallback} from './validate-video-frame';

packages/web-renderer/src/render-media-on-web.tsx

@@ -21,6 +21,8 @@ import type {
 	InferProps,
 } from './props-if-has-props';
 import {createFrame} from './take-screenshot';
+import {createThrottledProgressCallback} from './throttle-progress';
+import {validateVideoFrame, type OnFrameCallback} from './validate-video-frame';
 import {waitForReady} from './wait-for-ready';
 
 export type InputPropsIfHasProps<
@@ -78,6 +80,7 @@ type OptionalRenderMediaOnWebOptions<Schema extends AnyZodObject> = {
 	frameRange: FrameRange | null;
 	transparent: boolean;
 	onArtifact: OnArtifact | null;
+	onFrame: OnFrameCallback | null;
 };
 
 export type RenderMediaOnWebOptions<
@@ -96,13 +99,10 @@ type InternalRenderMediaOnWebOptions<
 
 // TODO: More containers
 // TODO: Audio
-// TODO: onFrame
 // TODO: Metadata
 // TODO: Validating inputs
 // TODO: Web file system API
 // TODO: Apply defaultCodec
-// TODO: Throttle onProgress
-// TODO: getStaticFiles()
 
 const internalRenderMediaOnWeb = async <
 	Schema extends AnyZodObject,
@@ -124,6 +124,7 @@ const internalRenderMediaOnWeb = async <
 	frameRange,
 	transparent,
 	onArtifact,
+	onFrame,
 }: InternalRenderMediaOnWebOptions<Schema, Props>) => {
 	const cleanupFns: (() => void)[] = [];
 	const format = containerToMediabunnyContainer(container);
@@ -249,6 +250,8 @@ const internalRenderMediaOnWeb = async <
 			encodedFrames: 0,
 		};
 
+		const throttledOnProgress = createThrottledProgressCallback(onProgress);
+
 		for (let i = realFrameRange[0]; i <= realFrameRange[1]; i++) {
 			timeUpdater.current?.update(i);
 			await waitForReady({
@@ -274,25 +277,42 @@ const internalRenderMediaOnWeb = async <
 				throw new Error('renderMediaOnWeb() was cancelled');
 			}
 
+			const timestamp = Math.round(
+				((i - realFrameRange[0]) / resolved.fps) * 1_000_000,
+			);
 			const videoFrame = new VideoFrame(imageData, {
-				timestamp: Math.round(
-					((i - realFrameRange[0]) / resolved.fps) * 1_000_000,
-				),
+				timestamp,
 			});
 			progress.renderedFrames++;
-			onProgress?.({...progress});
+			throttledOnProgress?.({...progress});
+
+			// Process frame through onFrame callback if provided
+			let frameToEncode = videoFrame;
+			if (onFrame) {
+				const returnedFrame = await onFrame(videoFrame);
+				frameToEncode = validateVideoFrame({
+					originalFrame: videoFrame,
+					returnedFrame,
+					expectedWidth: resolved.width,
+					expectedHeight: resolved.height,
+					expectedTimestamp: timestamp,
+				});
+			}
 
-			await videoSampleSource.add(new VideoSample(videoFrame));
+			await videoSampleSource.add(new VideoSample(frameToEncode));
 			progress.encodedFrames++;
-			onProgress?.({...progress});
+			throttledOnProgress?.({...progress});
 
-			videoFrame.close();
+			frameToEncode.close();
 
 			if (signal?.aborted) {
 				throw new Error('renderMediaOnWeb() was cancelled');
 			}
 		}
 
+		// Call progress one final time to ensure final state is reported
+		onProgress?.({...progress});
+
 		videoSampleSource.close();
 		await output.finalize();
 
@@ -328,5 +348,6 @@ export const renderMediaOnWeb = <
 		frameRange: options.frameRange ?? null,
 		transparent: options.transparent ?? false,
 		onArtifact: options.onArtifact ?? null,
+		onFrame: options.onFrame ?? null,
 	});
 };

packages/web-renderer/src/test/delay-render.test.tsx

@@ -1,4 +1,5 @@
 import {useEffect, useState} from 'react';
+import {flushSync} from 'react-dom';
 import {useDelayRender} from 'remotion';
 import {test} from 'vitest';
 import {renderStillOnWeb} from '../render-still-on-web';
@@ -12,7 +13,9 @@ test('should be able to use delayRender()', async () => {
 
 		useEffect(() => {
 			setTimeout(() => {
-				setData(true);
+				flushSync(() => {
+					setData(true);
+				});
 				continueRender(handle);
 			}, 50);
 		}, [continueRender, handle]);

packages/web-renderer/src/test/on-frame.test.tsx

@@ -0,0 +1,132 @@
+import {useCurrentFrame} from 'remotion';
+import {expect, test} from 'vitest';
+import {renderMediaOnWeb} from '../render-media-on-web';
+
+test('onFrame callback returning same frame should work', async (t) => {
+	if (t.task.file.projectName === 'webkit') {
+		t.skip();
+		return;
+	}
+
+	let frameCount = 0;
+
+	const Component: React.FC = () => {
+		const frame = useCurrentFrame();
+		return (
+			<div style={{width: 100, height: 100, backgroundColor: 'blue'}}>
+				Frame {frame}
+			</div>
+		);
+	};
+
+	await renderMediaOnWeb({
+		composition: {
+			component: Component,
+			id: 'on-frame-same-test',
+			width: 100,
+			height: 100,
+			fps: 30,
+			durationInFrames: 3,
+		},
+		inputProps: {},
+		onFrame: (frame) => {
+			frameCount++;
+			// Return the same frame
+			return frame;
+		},
+	});
+
+	expect(frameCount).toBe(3);
+});
+
+test('onFrame callback returning new frame with correct dimensions and timestamp should work', async (t) => {
+	if (t.task.file.projectName === 'webkit') {
+		t.skip();
+		return;
+	}
+
+	let frameCount = 0;
+
+	const Component: React.FC = () => {
+		const frame = useCurrentFrame();
+		return (
+			<div style={{width: 100, height: 100, backgroundColor: 'yellow'}}>
+				Frame {frame}
+			</div>
+		);
+	};
+
+	await renderMediaOnWeb({
+		composition: {
+			component: Component,
+			id: 'on-frame-new-valid-test',
+			width: 100,
+			height: 100,
+			fps: 30,
+			durationInFrames: 3,
+		},
+		inputProps: {},
+		onFrame: (frame) => {
+			frameCount++;
+			// Create a new canvas with the same dimensions
+			const canvas = new OffscreenCanvas(100, 100);
+			const ctx = canvas.getContext('2d')!;
+			ctx.fillStyle = 'purple';
+			ctx.fillRect(0, 0, 100, 100);
+
+			// Create new frame with same timestamp and dimensions
+			const newFrame = new VideoFrame(canvas, {
+				timestamp: frame.timestamp,
+			});
+
+			return newFrame;
+		},
+	});
+
+	expect(frameCount).toBe(3);
+});
+
+test('onFrame callback returning frame with wrong dimensions should throw', async (t) => {
+	if (t.task.file.projectName === 'webkit') {
+		t.skip();
+		return;
+	}
+
+	const Component: React.FC = () => {
+		const frame = useCurrentFrame();
+		return (
+			<div style={{width: 100, height: 100, backgroundColor: 'red'}}>
+				Frame {frame}
+			</div>
+		);
+	};
+
+	await expect(async () => {
+		await renderMediaOnWeb({
+			composition: {
+				component: Component,
+				id: 'on-frame-wrong-dimensions-test',
+				width: 100,
+				height: 100,
+				fps: 30,
+				durationInFrames: 2,
+			},
+			inputProps: {},
+			onFrame: (frame) => {
+				// Create a frame with wrong dimensions
+				const canvas = new OffscreenCanvas(200, 200);
+				const ctx = canvas.getContext('2d')!;
+				ctx.fillStyle = 'red';
+				ctx.fillRect(0, 0, 200, 200);
+
+				const newFrame = new VideoFrame(canvas, {
+					timestamp: frame.timestamp,
+				});
+
+				return newFrame;
+			},
+		});
+	}).rejects.toThrow(
+		/VideoFrame dimensions mismatch: expected 100x100, got 0x0/,
+	);
+});

packages/web-renderer/src/test/render-media.test.tsx

@@ -1,5 +1,5 @@
 import {interpolateColors, useCurrentFrame} from 'remotion';
-import {test} from 'vitest';
+import {expect, test} from 'vitest';
 import {renderMediaOnWeb} from '../render-media-on-web';
 
 test('should render media on web', async (t) => {
@@ -34,3 +34,66 @@ test('should render media on web', async (t) => {
 		inputProps: {},
 	});
 });
+
+test('should throttle onProgress callback to 250ms', async (t) => {
+	if (t.task.file.projectName === 'webkit') {
+		t.skip();
+		return;
+	}
+
+	const Component: React.FC = () => {
+		const frame = useCurrentFrame();
+		return (
+			<svg viewBox="0 0 100 100" style={{width: 400, height: 400}}>
+				<circle
+					cx="50"
+					cy="50"
+					r="50"
+					fill={interpolateColors(frame, [0, 30], ['red', 'blue'])}
+				/>
+			</svg>
+		);
+	};
+
+	const progressCalls: Array<{
+		time: number;
+		progress: {renderedFrames: number; encodedFrames: number};
+	}> = [];
+	const startTime = Date.now();
+
+	await renderMediaOnWeb({
+		composition: {
+			component: Component,
+			id: 'throttle-test',
+			width: 400,
+			height: 400,
+			fps: 30,
+			durationInFrames: 30,
+		},
+		inputProps: {},
+		onProgress: (progress) => {
+			progressCalls.push({
+				time: Date.now() - startTime,
+				progress: {...progress},
+			});
+		},
+	});
+
+	// Should have at least one progress call
+	expect(progressCalls.length).toBeGreaterThan(0);
+
+	// Final call should have all frames rendered and encoded
+	const finalCall = progressCalls[progressCalls.length - 1];
+	expect(finalCall.progress.renderedFrames).toBe(30);
+	expect(finalCall.progress.encodedFrames).toBe(30);
+
+	// Check that calls are throttled (if we have multiple calls)
+	if (progressCalls.length > 1) {
+		for (let i = 1; i < progressCalls.length - 1; i++) {
+			const timeDiff = progressCalls[i].time - progressCalls[i - 1].time;
+			// Allow some variance but should be around 250ms
+			// We use 200ms as lower bound to account for timing variations
+			expect(timeDiff).toBeGreaterThanOrEqual(200);
+		}
+	}
+});