Skip to content

Commit caec4f6

Browse files
JonnyBurgerJonnyBurger
authored
Merge pull request #5215 from remotion-dev/fast-path-mp4-fragmented
2 parents 3defe75 + 313fcbd commit caec4f6

28 files changed

+237
-98
lines changed

‎packages/media-parser/src/containers/iso-base-media/are-samples-complete.ts

+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import type {MoofBox} from '../../state/iso-base-media/precomputed-moof';
2+
import type {TfraBox} from './mfra/tfra';
3+
4+
export const areSamplesComplete = ({
5+
moofBoxes,
6+
tfraBoxes,
7+
}: {
8+
moofBoxes: MoofBox[];
9+
tfraBoxes: TfraBox[];
10+
}) => {
11+
if (moofBoxes.length === 0) {
12+
return true;
13+
}
14+
15+
return (
16+
tfraBoxes.length > 0 &&
17+
tfraBoxes.every((t) => t.entries.length === moofBoxes.length)
18+
);
19+
};

‎packages/media-parser/src/containers/iso-base-media/collect-sample-positions-from-moof-boxes.ts

+2-7
Original file line numberDiff line numberDiff line change
@@ -1,21 +1,16 @@
11
import {getSamplesFromMoof} from '../../samples-from-moof';
22
import type {MoofBox} from '../../state/iso-base-media/precomputed-moof';
3-
import type {TfraBox} from './mfra/tfra';
43
import type {TkhdBox} from './tkhd';
54

65
export const collectSamplePositionsFromMoofBoxes = ({
76
moofBoxes,
8-
tfraBoxes,
97
tkhdBox,
8+
isComplete,
109
}: {
1110
moofBoxes: MoofBox[];
12-
tfraBoxes: TfraBox[];
1311
tkhdBox: TkhdBox;
12+
isComplete: boolean;
1413
}) => {
15-
const isComplete =
16-
tfraBoxes.length > 0 &&
17-
tfraBoxes.every((t) => t.entries.length === moofBoxes.length);
18-
1914
const samplePositions = moofBoxes.map((m, index) => {
2015
const isLastFragment = index === moofBoxes.length - 1 && isComplete;
2116

‎packages/media-parser/src/containers/iso-base-media/find-track-to-seek.ts

+9-2
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ import type {SamplePosition} from '../../get-sample-positions';
22
import type {AudioTrack, OtherTrack, VideoTrack} from '../../get-tracks';
33
import type {IsoBaseMediaStructure} from '../../parse-result';
44
import type {StructureState} from '../../state/structure';
5+
import {areSamplesComplete} from './are-samples-complete';
56
import {getSamplePositionsFromTrack} from './get-sample-positions-from-track';
67
import type {TrakBox} from './trak/trak';
78
import {getMoofBoxes, getTfraBoxes} from './traversal';
@@ -15,7 +16,10 @@ export const findAnyTrackWithSamplePositions = (
1516
const {samplePositions} = getSamplePositionsFromTrack({
1617
trakBox: track.trakBox as TrakBox,
1718
moofBoxes: getMoofBoxes(struc.boxes),
18-
tfraBoxes: getTfraBoxes(struc),
19+
moofComplete: areSamplesComplete({
20+
moofBoxes: getMoofBoxes(struc.boxes),
21+
tfraBoxes: getTfraBoxes(struc.boxes),
22+
}),
1923
});
2024

2125
if (samplePositions.length === 0) {
@@ -48,7 +52,10 @@ export const findTrackToSeek = (
4852
const {samplePositions} = getSamplePositionsFromTrack({
4953
trakBox: firstVideoTrack.trakBox as TrakBox,
5054
moofBoxes: getMoofBoxes(struc.boxes),
51-
tfraBoxes: getTfraBoxes(struc),
55+
moofComplete: areSamplesComplete({
56+
moofBoxes: getMoofBoxes(struc.boxes),
57+
tfraBoxes: getTfraBoxes(struc.boxes),
58+
}),
5259
});
5360

5461
if (samplePositions.length === 0) {

‎packages/media-parser/src/containers/iso-base-media/get-keyframes.ts

+6-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
import {getTracksFromIsoBaseMedia} from '../../get-tracks';
22
import type {MediaParserKeyframe} from '../../options';
33
import type {ParserState} from '../../state/parser-state';
4+
import {areSamplesComplete} from './are-samples-complete';
45
import {getSamplePositionsFromTrack} from './get-sample-positions-from-track';
56
import type {TrakBox} from './trak/trak';
67
import {getMoofBoxes, getTfraBoxes} from './traversal';
@@ -17,14 +18,17 @@ export const getKeyframesFromIsoBaseMedia = (
1718
const structure = state.structure.getIsoStructure();
1819

1920
const moofBoxes = getMoofBoxes(structure.boxes);
20-
const tfraBoxes = getTfraBoxes(structure);
21+
const tfraBoxes = getTfraBoxes(structure.boxes);
2122

2223
const allSamples = videoTracks.map((t): MediaParserKeyframe[] => {
2324
const {timescale: ts} = t;
2425
const {samplePositions, isComplete} = getSamplePositionsFromTrack({
2526
trakBox: t.trakBox as TrakBox,
2627
moofBoxes,
27-
tfraBoxes,
28+
moofComplete: areSamplesComplete({
29+
moofBoxes,
30+
tfraBoxes,
31+
}),
2832
});
2933

3034
if (!isComplete) {

‎packages/media-parser/src/containers/iso-base-media/get-sample-positions-from-track.ts

+5-6
Original file line numberDiff line numberDiff line change
@@ -2,34 +2,33 @@ import type {SamplePosition} from '../../get-sample-positions';
22
import type {MoofBox} from '../../state/iso-base-media/precomputed-moof';
33
import {collectSamplePositionsFromMoofBoxes} from './collect-sample-positions-from-moof-boxes';
44
import {collectSamplePositionsFromTrak} from './collect-sample-positions-from-trak';
5-
import type {TfraBox} from './mfra/tfra';
65
import type {TrakBox} from './trak/trak';
76
import {getTkhdBox} from './traversal';
87

98
export const getSamplePositionsFromTrack = ({
109
trakBox,
1110
moofBoxes,
12-
tfraBoxes,
11+
moofComplete,
1312
}: {
1413
trakBox: TrakBox;
1514
moofBoxes: MoofBox[];
16-
tfraBoxes: TfraBox[];
15+
moofComplete: boolean;
1716
}): {samplePositions: SamplePosition[]; isComplete: boolean} => {
1817
const tkhdBox = getTkhdBox(trakBox);
1918
if (!tkhdBox) {
2019
throw new Error('Expected tkhd box in trak box');
2120
}
2221

2322
if (moofBoxes.length > 0) {
24-
const {isComplete, samplePositions} = collectSamplePositionsFromMoofBoxes({
23+
const {samplePositions} = collectSamplePositionsFromMoofBoxes({
2524
moofBoxes,
26-
tfraBoxes,
2725
tkhdBox,
26+
isComplete: moofComplete,
2827
});
2928

3029
return {
3130
samplePositions: samplePositions.map((s) => s.samples).flat(1),
32-
isComplete,
31+
isComplete: moofComplete,
3332
};
3433
}
3534

‎packages/media-parser/src/containers/iso-base-media/get-seeking-byte-from-fragmented-mp4.ts

+7-1
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import {
88
isByteInMediaSection,
99
} from '../../state/video-section';
1010
import type {SeekResolution} from '../../work-on-seek-request';
11+
import {areSamplesComplete} from './are-samples-complete';
1112
import {collectSamplePositionsFromMoofBoxes} from './collect-sample-positions-from-moof-boxes';
1213
import {findKeyframeBeforeTime} from './find-keyframe-before-time';
1314
import {getSamplePositionBounds} from './get-sample-position-bounds';
@@ -47,11 +48,16 @@ export const getSeekingByteFromFragmentedMp4 = async ({
4748
throw new Error('Expected tkhd box in trak box');
4849
}
4950

51+
const isComplete = areSamplesComplete({
52+
moofBoxes: info.moofBoxes,
53+
tfraBoxes: info.tfraBoxes,
54+
});
55+
5056
const {samplePositions: samplePositionsArray} =
5157
collectSamplePositionsFromMoofBoxes({
5258
moofBoxes: info.moofBoxes,
53-
tfraBoxes: info.tfraBoxes,
5459
tkhdBox,
60+
isComplete,
5561
});
5662

5763
Log.trace(

‎packages/media-parser/src/containers/iso-base-media/mdat/mdat.ts

+45-2
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
import {convertAudioOrVideoSampleToWebCodecsTimestamps} from '../../../convert-audio-or-video-sample';
22
import {getHasTracks} from '../../../get-tracks';
3+
import {Log} from '../../../log';
34
import type {FetchMoreData, Skip} from '../../../skip';
45
import {makeFetchMoreData, makeSkip} from '../../../skip';
56
import type {FlatSample} from '../../../state/iso-base-media/cached-sample-positions';
67
import {calculateFlatSamples} from '../../../state/iso-base-media/cached-sample-positions';
8+
import {getLastMoofBox} from '../../../state/iso-base-media/last-moof-box';
79
import {maySkipVideoData} from '../../../state/may-skip-video-data';
810
import type {ParserState} from '../../../state/parser-state';
911
import {getCurrentMediaSection} from '../../../state/video-section';
@@ -26,6 +28,22 @@ export const parseMdatSection = async (
2628

2729
// don't need mdat at all, can skip
2830
if (maySkipVideoData({state})) {
31+
const mfra = state.iso.mfra.getIfAlreadyLoaded();
32+
33+
if (mfra) {
34+
const lastMoof = getLastMoofBox(mfra);
35+
if (lastMoof && lastMoof > endOfMdat) {
36+
Log.verbose(
37+
state.logLevel,
38+
'Skipping to last moof',
39+
lastMoof,
40+
'end of mdat',
41+
endOfMdat,
42+
);
43+
return makeSkip(lastMoof);
44+
}
45+
}
46+
2947
return makeSkip(endOfMdat);
3048
}
3149

@@ -47,7 +65,10 @@ export const parseMdatSection = async (
4765
}
4866

4967
if (!state.iso.flatSamples.getSamples(mediaSection.start)) {
50-
const flattedSamples = calculateFlatSamples(state);
68+
const flattedSamples = calculateFlatSamples({
69+
state,
70+
mediaSectionStart: mediaSection.start,
71+
});
5172

5273
const calcedJumpMarks = calculateJumpMarks(flattedSamples, endOfMdat);
5374
state.iso.flatSamples.setJumpMarks(mediaSection.start, calcedJumpMarks);
@@ -66,7 +87,6 @@ export const parseMdatSection = async (
6687
const samplesWithIndex = flatSamples.find((sample) => {
6788
return sample.samplePosition.offset === iterator.counter.getOffset();
6889
});
69-
7090
if (!samplesWithIndex) {
7191
// There are various reasons why in mdat we find weird stuff:
7292
// - iphonevideo.hevc has a fake hoov atom which is not mapped
@@ -83,6 +103,14 @@ export const parseMdatSection = async (
83103

84104
// guess we reached the end!
85105
// iphonevideo.mov has extra padding here, so let's make sure to jump ahead
106+
107+
Log.verbose(
108+
state.logLevel,
109+
'Could not find sample at offset',
110+
iterator.counter.getOffset(),
111+
'skipping to end of mdat',
112+
);
113+
86114
return makeSkip(endOfMdat);
87115
}
88116

@@ -92,6 +120,14 @@ export const parseMdatSection = async (
92120
samplesWithIndex.samplePosition.size >
93121
state.contentLength
94122
) {
123+
Log.verbose(
124+
state.logLevel,
125+
"Sample is beyond the end of the file. Don't process it.",
126+
samplesWithIndex.samplePosition.offset +
127+
samplesWithIndex.samplePosition.size,
128+
endOfMdat,
129+
);
130+
95131
return makeSkip(endOfMdat);
96132
}
97133

@@ -138,6 +174,7 @@ export const parseMdatSection = async (
138174
// https://github.com/remotion-dev/remotion/issues/4680
139175
// In Chrome, we may not treat recovery points as keyframes
140176
// otherwise "a keyframe is required after flushing"
177+
141178
const nalUnitType = bytes[4] & 0b00011111;
142179
let isRecoveryPoint = false;
143180
// SEI (Supplemental enhancement information)
@@ -169,6 +206,12 @@ export const parseMdatSection = async (
169206

170207
const jump = jumpMarks.find((j) => j.afterSampleWithOffset === offset);
171208
if (jump) {
209+
Log.verbose(
210+
state.logLevel,
211+
'Found jump mark',
212+
jump.jumpToOffset,
213+
'skipping to jump mark',
214+
);
172215
return makeSkip(jump.jumpToOffset);
173216
}
174217

‎packages/media-parser/src/containers/iso-base-media/mfra/get-mfro-atom.ts

+1
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ export const getMfroAtom = async ({
2727
logLevel,
2828
prefetchCache,
2929
});
30+
3031
const {value} = await result.reader.reader.read();
3132
if (!value) {
3233
return null;

‎packages/media-parser/src/containers/iso-base-media/process-box.ts

+7-7
Original file line numberDiff line numberDiff line change
@@ -139,14 +139,14 @@ export const processBox = async ({
139139
if (boxType === 'ftyp') {
140140
return {
141141
type: 'box',
142-
box: await parseFtyp({iterator, size: boxSize, offset: fileOffset}),
142+
box: parseFtyp({iterator, size: boxSize, offset: fileOffset}),
143143
};
144144
}
145145

146146
if (boxType === 'colr') {
147147
return {
148148
type: 'box',
149-
box: await parseColorParameterBox({
149+
box: parseColorParameterBox({
150150
iterator,
151151
size: boxSize,
152152
}),
@@ -156,28 +156,28 @@ export const processBox = async ({
156156
if (boxType === 'mvhd') {
157157
return {
158158
type: 'box',
159-
box: await parseMvhd({iterator, offset: fileOffset, size: boxSize}),
159+
box: parseMvhd({iterator, offset: fileOffset, size: boxSize}),
160160
};
161161
}
162162

163163
if (boxType === 'tkhd') {
164164
return {
165165
type: 'box',
166-
box: await parseTkhd({iterator, offset: fileOffset, size: boxSize}),
166+
box: parseTkhd({iterator, offset: fileOffset, size: boxSize}),
167167
};
168168
}
169169

170170
if (boxType === 'trun') {
171171
return {
172172
type: 'box',
173-
box: await parseTrun({iterator, offset: fileOffset, size: boxSize}),
173+
box: parseTrun({iterator, offset: fileOffset, size: boxSize}),
174174
};
175175
}
176176

177177
if (boxType === 'tfdt') {
178178
return {
179179
type: 'box',
180-
box: await parseTfdt({iterator, size: boxSize, offset: fileOffset}),
180+
box: parseTfdt({iterator, size: boxSize, offset: fileOffset}),
181181
};
182182
}
183183

@@ -464,7 +464,7 @@ export const processBox = async ({
464464
}
465465

466466
if (boxType === 'moof') {
467-
onlyIfMoovAtomExpected?.isoState?.mfra.triggerLoad();
467+
await onlyIfMoovAtomExpected?.isoState?.mfra.triggerLoad();
468468
}
469469

470470
if (

‎packages/media-parser/src/containers/iso-base-media/seeking-hints.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ export const getSeekingHintsFromMp4 = ({
3333
]);
3434
const tfraBoxes = deduplicateTfraBoxesByOffset([
3535
...isoState.tfra.getTfraBoxes(),
36-
...getTfraBoxes(structure),
36+
...getTfraBoxes(structure.boxes),
3737
]);
3838

3939
if (!moovAtom) {

‎packages/media-parser/src/containers/iso-base-media/traversal.ts

+13-7
Original file line numberDiff line numberDiff line change
@@ -309,18 +309,24 @@ export const getTrunBoxes = (segment: IsoBaseMediaBox): TrunBox[] => {
309309
return trunBoxes as TrunBox[];
310310
};
311311

312-
export const getTfraBoxes = (structure: IsoBaseMediaStructure): TfraBox[] => {
313-
const mfraBox = structure.boxes.find(
312+
export const getTfraBoxesFromMfraBoxChildren = (
313+
mfraBoxChildren: IsoBaseMediaBox[],
314+
): TfraBox[] => {
315+
const tfraBoxes = mfraBoxChildren.filter(
316+
(b) => b.type === 'tfra-box',
317+
) as TfraBox[];
318+
319+
return tfraBoxes;
320+
};
321+
322+
export const getTfraBoxes = (structure: IsoBaseMediaBox[]): TfraBox[] => {
323+
const mfraBox = structure.find(
314324
(b) => b.type === 'regular-box' && b.boxType === 'mfra',
315325
) as RegularBox | null;
316326

317327
if (!mfraBox) {
318328
return [];
319329
}
320330

321-
const tfraBoxes = mfraBox.children.filter(
322-
(b) => b.type === 'tfra-box',
323-
) as TfraBox[];
324-
325-
return tfraBoxes;
331+
return getTfraBoxesFromMfraBoxChildren(mfraBox.children);
326332
};

0 commit comments

Comments
 (0)