Merge pull request #5215 from remotion-dev/fast-path-mp4-fragmented

Author: JonnyBurger

packages/media-parser/src/containers/iso-base-media/are-samples-complete.ts

@@ -0,0 +1,19 @@
+import type {MoofBox} from '../../state/iso-base-media/precomputed-moof';
+import type {TfraBox} from './mfra/tfra';
+
+export const areSamplesComplete = ({
+	moofBoxes,
+	tfraBoxes,
+}: {
+	moofBoxes: MoofBox[];
+	tfraBoxes: TfraBox[];
+}) => {
+	if (moofBoxes.length === 0) {
+		return true;
+	}
+
+	return (
+		tfraBoxes.length > 0 &&
+		tfraBoxes.every((t) => t.entries.length === moofBoxes.length)
+	);
+};

packages/media-parser/src/containers/iso-base-media/collect-sample-positions-from-moof-boxes.ts

@@ -1,21 +1,16 @@
 import {getSamplesFromMoof} from '../../samples-from-moof';
 import type {MoofBox} from '../../state/iso-base-media/precomputed-moof';
-import type {TfraBox} from './mfra/tfra';
 import type {TkhdBox} from './tkhd';
 
 export const collectSamplePositionsFromMoofBoxes = ({
 	moofBoxes,
-	tfraBoxes,
 	tkhdBox,
+	isComplete,
 }: {
 	moofBoxes: MoofBox[];
-	tfraBoxes: TfraBox[];
 	tkhdBox: TkhdBox;
+	isComplete: boolean;
 }) => {
-	const isComplete =
-		tfraBoxes.length > 0 &&
-		tfraBoxes.every((t) => t.entries.length === moofBoxes.length);
-
 	const samplePositions = moofBoxes.map((m, index) => {
 		const isLastFragment = index === moofBoxes.length - 1 && isComplete;
 

packages/media-parser/src/containers/iso-base-media/find-track-to-seek.ts

@@ -2,6 +2,7 @@ import type {SamplePosition} from '../../get-sample-positions';
 import type {AudioTrack, OtherTrack, VideoTrack} from '../../get-tracks';
 import type {IsoBaseMediaStructure} from '../../parse-result';
 import type {StructureState} from '../../state/structure';
+import {areSamplesComplete} from './are-samples-complete';
 import {getSamplePositionsFromTrack} from './get-sample-positions-from-track';
 import type {TrakBox} from './trak/trak';
 import {getMoofBoxes, getTfraBoxes} from './traversal';
@@ -15,7 +16,10 @@ export const findAnyTrackWithSamplePositions = (
 			const {samplePositions} = getSamplePositionsFromTrack({
 				trakBox: track.trakBox as TrakBox,
 				moofBoxes: getMoofBoxes(struc.boxes),
-				tfraBoxes: getTfraBoxes(struc),
+				moofComplete: areSamplesComplete({
+					moofBoxes: getMoofBoxes(struc.boxes),
+					tfraBoxes: getTfraBoxes(struc.boxes),
+				}),
 			});
 
 			if (samplePositions.length === 0) {
@@ -48,7 +52,10 @@ export const findTrackToSeek = (
 	const {samplePositions} = getSamplePositionsFromTrack({
 		trakBox: firstVideoTrack.trakBox as TrakBox,
 		moofBoxes: getMoofBoxes(struc.boxes),
-		tfraBoxes: getTfraBoxes(struc),
+		moofComplete: areSamplesComplete({
+			moofBoxes: getMoofBoxes(struc.boxes),
+			tfraBoxes: getTfraBoxes(struc.boxes),
+		}),
 	});
 
 	if (samplePositions.length === 0) {

packages/media-parser/src/containers/iso-base-media/get-keyframes.ts

@@ -1,6 +1,7 @@
 import {getTracksFromIsoBaseMedia} from '../../get-tracks';
 import type {MediaParserKeyframe} from '../../options';
 import type {ParserState} from '../../state/parser-state';
+import {areSamplesComplete} from './are-samples-complete';
 import {getSamplePositionsFromTrack} from './get-sample-positions-from-track';
 import type {TrakBox} from './trak/trak';
 import {getMoofBoxes, getTfraBoxes} from './traversal';
@@ -17,14 +18,17 @@ export const getKeyframesFromIsoBaseMedia = (
 	const structure = state.structure.getIsoStructure();
 
 	const moofBoxes = getMoofBoxes(structure.boxes);
-	const tfraBoxes = getTfraBoxes(structure);
+	const tfraBoxes = getTfraBoxes(structure.boxes);
 
 	const allSamples = videoTracks.map((t): MediaParserKeyframe[] => {
 		const {timescale: ts} = t;
 		const {samplePositions, isComplete} = getSamplePositionsFromTrack({
 			trakBox: t.trakBox as TrakBox,
 			moofBoxes,
-			tfraBoxes,
+			moofComplete: areSamplesComplete({
+				moofBoxes,
+				tfraBoxes,
+			}),
 		});
 
 		if (!isComplete) {

packages/media-parser/src/containers/iso-base-media/get-sample-positions-from-track.ts

@@ -2,34 +2,33 @@ import type {SamplePosition} from '../../get-sample-positions';
 import type {MoofBox} from '../../state/iso-base-media/precomputed-moof';
 import {collectSamplePositionsFromMoofBoxes} from './collect-sample-positions-from-moof-boxes';
 import {collectSamplePositionsFromTrak} from './collect-sample-positions-from-trak';
-import type {TfraBox} from './mfra/tfra';
 import type {TrakBox} from './trak/trak';
 import {getTkhdBox} from './traversal';
 
 export const getSamplePositionsFromTrack = ({
 	trakBox,
 	moofBoxes,
-	tfraBoxes,
+	moofComplete,
 }: {
 	trakBox: TrakBox;
 	moofBoxes: MoofBox[];
-	tfraBoxes: TfraBox[];
+	moofComplete: boolean;
 }): {samplePositions: SamplePosition[]; isComplete: boolean} => {
 	const tkhdBox = getTkhdBox(trakBox);
 	if (!tkhdBox) {
 		throw new Error('Expected tkhd box in trak box');
 	}
 
 	if (moofBoxes.length > 0) {
-		const {isComplete, samplePositions} = collectSamplePositionsFromMoofBoxes({
+		const {samplePositions} = collectSamplePositionsFromMoofBoxes({
 			moofBoxes,
-			tfraBoxes,
 			tkhdBox,
+			isComplete: moofComplete,
 		});
 
 		return {
 			samplePositions: samplePositions.map((s) => s.samples).flat(1),
-			isComplete,
+			isComplete: moofComplete,
 		};
 	}
 

packages/media-parser/src/containers/iso-base-media/get-seeking-byte-from-fragmented-mp4.ts

@@ -8,6 +8,7 @@ import {
 	isByteInMediaSection,
 } from '../../state/video-section';
 import type {SeekResolution} from '../../work-on-seek-request';
+import {areSamplesComplete} from './are-samples-complete';
 import {collectSamplePositionsFromMoofBoxes} from './collect-sample-positions-from-moof-boxes';
 import {findKeyframeBeforeTime} from './find-keyframe-before-time';
 import {getSamplePositionBounds} from './get-sample-position-bounds';
@@ -47,11 +48,16 @@ export const getSeekingByteFromFragmentedMp4 = async ({
 		throw new Error('Expected tkhd box in trak box');
 	}
 
+	const isComplete = areSamplesComplete({
+		moofBoxes: info.moofBoxes,
+		tfraBoxes: info.tfraBoxes,
+	});
+
 	const {samplePositions: samplePositionsArray} =
 		collectSamplePositionsFromMoofBoxes({
 			moofBoxes: info.moofBoxes,
-			tfraBoxes: info.tfraBoxes,
 			tkhdBox,
+			isComplete,
 		});
 
 	Log.trace(

packages/media-parser/src/containers/iso-base-media/mdat/mdat.ts

@@ -1,9 +1,11 @@
 import {convertAudioOrVideoSampleToWebCodecsTimestamps} from '../../../convert-audio-or-video-sample';
 import {getHasTracks} from '../../../get-tracks';
+import {Log} from '../../../log';
 import type {FetchMoreData, Skip} from '../../../skip';
 import {makeFetchMoreData, makeSkip} from '../../../skip';
 import type {FlatSample} from '../../../state/iso-base-media/cached-sample-positions';
 import {calculateFlatSamples} from '../../../state/iso-base-media/cached-sample-positions';
+import {getLastMoofBox} from '../../../state/iso-base-media/last-moof-box';
 import {maySkipVideoData} from '../../../state/may-skip-video-data';
 import type {ParserState} from '../../../state/parser-state';
 import {getCurrentMediaSection} from '../../../state/video-section';
@@ -26,6 +28,22 @@ export const parseMdatSection = async (
 
 	// don't need mdat at all, can skip
 	if (maySkipVideoData({state})) {
+		const mfra = state.iso.mfra.getIfAlreadyLoaded();
+
+		if (mfra) {
+			const lastMoof = getLastMoofBox(mfra);
+			if (lastMoof && lastMoof > endOfMdat) {
+				Log.verbose(
+					state.logLevel,
+					'Skipping to last moof',
+					lastMoof,
+					'end of mdat',
+					endOfMdat,
+				);
+				return makeSkip(lastMoof);
+			}
+		}
+
 		return makeSkip(endOfMdat);
 	}
 
@@ -47,7 +65,10 @@ export const parseMdatSection = async (
 	}
 
 	if (!state.iso.flatSamples.getSamples(mediaSection.start)) {
-		const flattedSamples = calculateFlatSamples(state);
+		const flattedSamples = calculateFlatSamples({
+			state,
+			mediaSectionStart: mediaSection.start,
+		});
 
 		const calcedJumpMarks = calculateJumpMarks(flattedSamples, endOfMdat);
 		state.iso.flatSamples.setJumpMarks(mediaSection.start, calcedJumpMarks);
@@ -66,7 +87,6 @@ export const parseMdatSection = async (
 	const samplesWithIndex = flatSamples.find((sample) => {
 		return sample.samplePosition.offset === iterator.counter.getOffset();
 	});
-
 	if (!samplesWithIndex) {
 		// There are various reasons why in mdat we find weird stuff:
 		// - iphonevideo.hevc has a fake hoov atom which is not mapped
@@ -83,6 +103,14 @@ export const parseMdatSection = async (
 
 		// guess we reached the end!
 		// iphonevideo.mov has extra padding here, so let's make sure to jump ahead
+
+		Log.verbose(
+			state.logLevel,
+			'Could not find sample at offset',
+			iterator.counter.getOffset(),
+			'skipping to end of mdat',
+		);
+
 		return makeSkip(endOfMdat);
 	}
 
@@ -92,6 +120,14 @@ export const parseMdatSection = async (
 			samplesWithIndex.samplePosition.size >
 		state.contentLength
 	) {
+		Log.verbose(
+			state.logLevel,
+			"Sample is beyond the end of the file. Don't process it.",
+			samplesWithIndex.samplePosition.offset +
+				samplesWithIndex.samplePosition.size,
+			endOfMdat,
+		);
+
 		return makeSkip(endOfMdat);
 	}
 
@@ -138,6 +174,7 @@ export const parseMdatSection = async (
 		// https://github.com/remotion-dev/remotion/issues/4680
 		// In Chrome, we may not treat recovery points as keyframes
 		// otherwise "a keyframe is required after flushing"
+
 		const nalUnitType = bytes[4] & 0b00011111;
 		let isRecoveryPoint = false;
 		// SEI (Supplemental enhancement information)
@@ -169,6 +206,12 @@ export const parseMdatSection = async (
 
 	const jump = jumpMarks.find((j) => j.afterSampleWithOffset === offset);
 	if (jump) {
+		Log.verbose(
+			state.logLevel,
+			'Found jump mark',
+			jump.jumpToOffset,
+			'skipping to jump mark',
+		);
 		return makeSkip(jump.jumpToOffset);
 	}
 

packages/media-parser/src/containers/iso-base-media/mfra/get-mfro-atom.ts

@@ -27,6 +27,7 @@ export const getMfroAtom = async ({
 		logLevel,
 		prefetchCache,
 	});
+
 	const {value} = await result.reader.reader.read();
 	if (!value) {
 		return null;

packages/media-parser/src/containers/iso-base-media/process-box.ts

@@ -139,14 +139,14 @@ export const processBox = async ({
 	if (boxType === 'ftyp') {
 		return {
 			type: 'box',
-			box: await parseFtyp({iterator, size: boxSize, offset: fileOffset}),
+			box: parseFtyp({iterator, size: boxSize, offset: fileOffset}),
 		};
 	}
 
 	if (boxType === 'colr') {
 		return {
 			type: 'box',
-			box: await parseColorParameterBox({
+			box: parseColorParameterBox({
 				iterator,
 				size: boxSize,
 			}),
@@ -156,28 +156,28 @@ export const processBox = async ({
 	if (boxType === 'mvhd') {
 		return {
 			type: 'box',
-			box: await parseMvhd({iterator, offset: fileOffset, size: boxSize}),
+			box: parseMvhd({iterator, offset: fileOffset, size: boxSize}),
 		};
 	}
 
 	if (boxType === 'tkhd') {
 		return {
 			type: 'box',
-			box: await parseTkhd({iterator, offset: fileOffset, size: boxSize}),
+			box: parseTkhd({iterator, offset: fileOffset, size: boxSize}),
 		};
 	}
 
 	if (boxType === 'trun') {
 		return {
 			type: 'box',
-			box: await parseTrun({iterator, offset: fileOffset, size: boxSize}),
+			box: parseTrun({iterator, offset: fileOffset, size: boxSize}),
 		};
 	}
 
 	if (boxType === 'tfdt') {
 		return {
 			type: 'box',
-			box: await parseTfdt({iterator, size: boxSize, offset: fileOffset}),
+			box: parseTfdt({iterator, size: boxSize, offset: fileOffset}),
 		};
 	}
 
@@ -464,7 +464,7 @@ export const processBox = async ({
 	}
 
 	if (boxType === 'moof') {
-		onlyIfMoovAtomExpected?.isoState?.mfra.triggerLoad();
+		await onlyIfMoovAtomExpected?.isoState?.mfra.triggerLoad();
 	}
 
 	if (

packages/media-parser/src/containers/iso-base-media/seeking-hints.ts

@@ -33,7 +33,7 @@ export const getSeekingHintsFromMp4 = ({
 	]);
 	const tfraBoxes = deduplicateTfraBoxesByOffset([
 		...isoState.tfra.getTfraBoxes(),
-		...getTfraBoxes(structure),
+		...getTfraBoxes(structure.boxes),
 	]);
 
 	if (!moovAtom) {

packages/media-parser/src/containers/iso-base-media/traversal.ts

@@ -309,18 +309,24 @@ export const getTrunBoxes = (segment: IsoBaseMediaBox): TrunBox[] => {
 	return trunBoxes as TrunBox[];
 };
 
-export const getTfraBoxes = (structure: IsoBaseMediaStructure): TfraBox[] => {
-	const mfraBox = structure.boxes.find(
+export const getTfraBoxesFromMfraBoxChildren = (
+	mfraBoxChildren: IsoBaseMediaBox[],
+): TfraBox[] => {
+	const tfraBoxes = mfraBoxChildren.filter(
+		(b) => b.type === 'tfra-box',
+	) as TfraBox[];
+
+	return tfraBoxes;
+};
+
+export const getTfraBoxes = (structure: IsoBaseMediaBox[]): TfraBox[] => {
+	const mfraBox = structure.find(
 		(b) => b.type === 'regular-box' && b.boxType === 'mfra',
 	) as RegularBox | null;
 
 	if (!mfraBox) {
 		return [];
 	}
 
-	const tfraBoxes = mfraBox.children.filter(
-		(b) => b.type === 'tfra-box',
-	) as TfraBox[];
-
-	return tfraBoxes;
+	return getTfraBoxesFromMfraBoxChildren(mfraBox.children);
 };