diff --git a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts index 49cfa78123..d38226c640 100644 --- a/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/media-routes-real-world.integration.test.ts @@ -20,6 +20,7 @@ import * as containerCpu from "../../lib/container-cpu"; import * as containerMemory from "../../lib/container-memory"; import type { Job, JobProgress } from "../../lib/job-manager"; import { probeVideoFile } from "../../lib/media-probe"; +import * as mediaVideo from "../../lib/media-video"; import * as recordingVerification from "../../lib/recording-verification"; const FIXTURES_DIR = join(import.meta.dir, "..", "fixtures"); @@ -48,7 +49,12 @@ const sourceReads: { const uploadConditions: (string | null)[] = []; const multipartCallbacks: { action: string; payload: unknown }[] = []; let rejectMultipartSigning = false; -let sourceFault: "changed" | "missing" | "corrupt" | undefined; +let sourceFault: + | "changed" + | "missing" + | "corrupt" + | "corrupt-audio" + | undefined; let corruptRecordingReadback = false; let transientFixtureFailures = 0; let permanentFixtureFailures = 0; @@ -294,7 +300,11 @@ beforeAll(async () => { ifMatch: request.headers.get("if-match"), verification: request.headers.get("x-cap-recording-verification"), }); - const affected = url.pathname.endsWith("video-segment.m4s"); + const affected = url.pathname.endsWith( + sourceFault === "corrupt-audio" + ? "audio-segment.m4s" + : "video-segment.m4s", + ); if (affected && sourceFault === "missing") return new Response(null, { status: 404 }); if ( @@ -305,7 +315,8 @@ beforeAll(async () => { return new Response(null, { status: 412 }); return new Response( Uint8Array.from( - affected && sourceFault === "corrupt" + affected && + (sourceFault === "corrupt" || sourceFault === "corrupt-audio") ? new Uint8Array(source.byteLength) : source, ).buffer, @@ -460,6 +471,30 @@ afterAll(() => { }); describe("media routes real-world integration tests", () => { + test("reports a failed output write without classifying pinned source media as invalid", async () => { + const mux = spyOn(mediaVideo, "muxMediaTracksToMp4").mockRejectedValue( + new Error("Could not write output header"), + ); + let jobId: string | undefined; + try { + const response = await app.fetch( + mediaPostRequest( + "/video/mux-segments", + fencedMuxRequest("failed-output-write"), + ), + ); + expect(response.status).toBe(200); + jobId = ((await response.json()) as { jobId: string }).jobId; + const job = await waitForTerminalJob(jobId); + expect(job.phase).toBe("error"); + expect(job.errorCode).toBe("output-invalid"); + expect(job.recordingVerification).toBeUndefined(); + expect(uploadConditions).toHaveLength(0); + } finally { + mux.mockRestore(); + if (jobId) deleteJob(jobId); + } + }); test("cancels segmented work at its total deadline without waiting for job cleanup", async () => { const originalSetTimeout = globalThis.setTimeout; const shortenedDeadline = new Proxy(originalSetTimeout, { @@ -550,7 +585,7 @@ describe("media routes real-world integration tests", () => { recordingVerification, "inspectRecordingSources", ); - const localDecode = spyOn(recordingVerification, "verifyRecording"); + const localDecode = spyOn(recordingVerification, "verifyRemuxedRecording"); const remoteDecode = spyOn(recordingVerification, "verifyRemoteRecording"); const bytesOnly = spyOn( recordingVerification, @@ -574,7 +609,7 @@ describe("media routes real-world integration tests", () => { expect(job.attemptId).toBe(body.attemptId); expect(job.inventorySha256).toBe(body.inventorySha256); expect(job.metadata?.duration).toBeCloseTo(1, 3); - expect(sourceDecode).toHaveBeenCalledTimes(1); + expect(sourceDecode.mock.calls.length).toBeLessThanOrEqual(1); expect(localDecode).toHaveBeenCalledTimes(1); expect(remoteDecode).not.toHaveBeenCalled(); expect(bytesOnly).toHaveBeenCalledTimes(1); @@ -682,7 +717,7 @@ describe("media routes real-world integration tests", () => { 30_000, ); - test.each(["changed", "missing", "corrupt"] as const)( + test.each(["changed", "missing", "corrupt", "corrupt-audio"] as const)( "withholds upload and proof after a pinned source is %s", async (fault) => { sourceFault = fault; @@ -696,7 +731,7 @@ describe("media routes real-world integration tests", () => { const job = await waitForTerminalJob(jobId); expect(job.phase).toBe("error"); expect(job.errorCode).toBe( - fault === "corrupt" ? "source-invalid" : `source-${fault}`, + fault.startsWith("corrupt") ? "source-invalid" : `source-${fault}`, ); expect(job.recordingVerification).toBeUndefined(); expect(uploadConditions).toHaveLength(0); diff --git a/apps/media-server/src/__tests__/lib/media-transfer.test.ts b/apps/media-server/src/__tests__/lib/media-transfer.test.ts index 5fa0b37e04..26d5a734f4 100644 --- a/apps/media-server/src/__tests__/lib/media-transfer.test.ts +++ b/apps/media-server/src/__tests__/lib/media-transfer.test.ts @@ -115,6 +115,42 @@ describe("bounded revision downloads", () => { expect(received).toBe(content.length); expect(calls).toBe(2); }); + test.each(["connection", "unavailable"])( + "retains downloaded bytes across a %s failure before response headers", + async (fault) => { + const path = await destination(); + let calls = 0; + let received = 0; + await downloadDriveRevision(target(), path, { + fetcher: fetcher((_input, init) => { + calls++; + if (calls === 1) + return new Response(content.subarray(0, 12), { + headers: { "Content-Length": String(content.length) }, + }); + expect(new Headers(init?.headers).get("range")).toBe("bytes=12-"); + if (calls === 2) { + if (fault === "connection") + throw new TypeError("Connection failed"); + return new Response(null, { status: 503 }); + } + return new Response(content.subarray(12), { + status: 206, + headers: { + "Content-Length": String(content.length - 12), + "Content-Range": `bytes 12-${content.length - 1}/${content.length}`, + }, + }); + }), + onBytes: (bytes) => { + received += bytes; + }, + }); + expect(await readFile(path)).toEqual(content); + expect(received).toBe(content.length); + expect(calls).toBe(3); + }, + ); test("rejects corruption and removes the incomplete file", async () => { const path = await destination(); await expect( diff --git a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts index a402406077..0d11a2fb7f 100644 --- a/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts +++ b/apps/media-server/src/__tests__/lib/recording-verification.integration.test.ts @@ -14,6 +14,10 @@ import { join } from "node:path"; import { Readable } from "node:stream"; import { EncodedPacketSink, FilePathSource, Input, MP4 } from "mediabunny"; import { muxMediaTracksToMp4 } from "../../lib/media-video"; +import { + proveRecordingPackets, + readRecordingAudioTail, +} from "../../lib/recording-packet-proof"; import { RecordingTimingError, readRecordingVideoTiming, @@ -24,6 +28,7 @@ import { verifyRecording, verifyRemoteRecording, verifyRemoteRecordingBytes, + verifyRemuxedRecording, } from "../../lib/recording-verification"; const FIXTURES = join(import.meta.dir, "..", "fixtures"); @@ -486,6 +491,246 @@ afterAll(async () => { if (directory) await rm(directory, { recursive: true, force: true }); }); +describe("encoded recording preservation", () => { + test.skipIf(process.platform === "win32")( + "kills and joins a stalled audio inspector on cancellation and timeout", + async () => { + for (const mode of ["cancel", "timeout"]) { + const path = join(directory, `stalled-audio-${mode}`); + await run(["mkfifo", path]); + const controller = new AbortController(); + const pending = readRecordingAudioTail(path, controller.signal); + pending.catch(() => {}); + let pids: number[] = []; + let timer: ReturnType | undefined; + try { + for (let attempt = 0; attempt < 100 && !pids.length; attempt++) { + pids = await decoderPids(path); + if (!pids.length) await Bun.sleep(10); + } + expect(pids).toHaveLength(1); + await Bun.sleep(100); + const started = performance.now(); + if (mode === "timeout") + timer = setTimeout( + () => + controller.abort(new DOMException("Timed out", "TimeoutError")), + 50, + ); + else controller.abort(); + await expect( + Promise.race([ + pending, + Bun.sleep(2000).then(() => { + throw new Error("Audio inspection ignored cancellation"); + }), + ]), + ).rejects.toMatchObject({ retryable: true }); + expect(performance.now() - started).toBeLessThan(2000); + expect(await decoderPids(path)).toEqual([]); + } finally { + if (timer) clearTimeout(timer); + controller.abort(); + for (const pid of pids) { + try { + process.kill(pid, "SIGKILL"); + } catch {} + } + await pending.catch(() => {}); + } + } + }, + ); + test("retains the processing deadline reason when muxing is already cancelled", async () => { + const reason = new Error("Recording processing timed out"); + await expect( + muxMediaTracksToMp4( + silent, + silent, + join(directory, "cancelled-mux.mp4"), + AbortSignal.abort(reason), + ), + ).rejects.toBe(reason); + }); + test("distinguishes malformed audio from unavailable local timing reads", async () => { + await expect( + readRecordingAudioTail( + join(directory, "absent-audio.mp4"), + AbortSignal.timeout(5000), + ), + ).rejects.toMatchObject({ retryable: true }); + await expect( + readRecordingAudioTail(silent, AbortSignal.abort()), + ).rejects.toMatchObject({ retryable: true }); + const malformed = join(directory, "malformed-audio.mp4"); + await writeFile(malformed, Buffer.alloc(32)); + await expect( + readRecordingAudioTail(malformed, AbortSignal.timeout(5000)), + ).rejects.toMatchObject({ retryable: false }); + }); + test("uses decoded source evidence for tied terminal video samples", async () => { + const input = await tiedTimestampSource("packet-tied-terminal.mp4", 2); + const output = join(directory, "packet-tied-terminal-output.mp4"); + await muxMediaTracksToMp4(input, silent, output); + const verified = await verifyRemuxedRecording(input, silent, output, { + requireAudio: true, + }); + expect(verified.sourcePreserved).toBe(true); + expect(verified.integrity).toBeDefined(); + expect(verified.video.frameCount).toBe(40); + }); + test("preserves the stored looped AAC tail for one complete output decode", async () => { + const input = join(directory, "looped-audio.mp4"); + const output = join(directory, "looped-audio-remux.mp4"); + await run([ + "ffmpeg", + "-v", + "error", + "-stream_loop", + "2", + "-i", + silent, + "-c", + "copy", + input, + ]); + await muxMediaTracksToMp4(input, input, output); + const verified = await verifyRemuxedRecording(input, input, output, { + requireAudio: true, + }); + expect(verified.fullDecode).toBe(true); + expect(verified.sourcePreserved).toBe(true); + expect(verified.integrity).toBeUndefined(); + const source = await inspectRecordingSources(input, input); + expect(verified.audio).toEqual(source.audio); + expect(verified.video).toEqual(source.video); + }); + test("refuses identical corrupt packets rather than treating preservation as decodability", async () => { + await expect( + verifyRemuxedRecording(corruptTail, corruptTail, corruptTail, { + requireAudio: true, + }), + ).rejects.toThrow(); + }); + test("binds backward presentation timestamps without changing the recording", async () => { + const samples = bFrameSamples.map((sample) => ({ ...sample })); + samples[13].pts -= 6000; + const input = join(directory, "packet-backward-pts.mp4"); + await writeFile( + input, + Buffer.concat([ + bFrameInit, + ...samples.map((sample, index) => sampleFragment([sample], index + 1)), + ]), + ); + const output = join(directory, "packet-backward-output.mp4"); + await muxMediaTracksToMp4(input, null, output); + const result = await verifyRemuxedRecording(input, null, output, { + requireAudio: false, + }); + expect(result.sourcePreserved).toBe(true); + expect(result.video.frameCount).toBe(samples.length); + expect(result.integrity).toBeUndefined(); + }); + test("applies one deadline to packet inspection and decode", async () => { + await expect( + verifyRemuxedRecording(silent, silent, silent, { + requireAudio: true, + timeoutMs: 1, + }), + ).rejects.toThrow(); + expect(await decoderPids(silent)).toEqual([]); + }); + + test.each([true, false])( + "decodes preserved packets once with audio=%s", + async (audio) => { + const input = join( + FIXTURES, + audio ? "test-with-audio.mp4" : "test-no-audio.mp4", + ); + const output = join(directory, `packet-proof-${audio}.mp4`); + await muxMediaTracksToMp4(input, audio ? input : null, output); + await proveRecordingPackets( + input, + audio ? input : null, + output, + AbortSignal.timeout(5000), + ); + const verified = await verifyRemuxedRecording( + input, + audio ? input : null, + output, + { requireAudio: audio }, + ); + expect(verified.fullDecode).toBe(true); + expect(verified.sourcePreserved).toBe(true); + expect(Boolean(verified.audio)).toBe(audio); + expect(verified.integrity).toBeUndefined(); + }, + ); + test("rejects changed source bytes", async () => { + await expect( + proveRecordingPackets( + silent, + silent, + corruptTail, + AbortSignal.timeout(5000), + ), + ).rejects.toThrow(); + await expect( + verifyRemuxedRecording(silent, silent, corruptTail, { + requireAudio: true, + }), + ).rejects.toThrow(); + }); + test("preserves a shorter audio track independently of the video", async () => { + const output = join(directory, "proof-short-audio.mp4"); + await muxMediaTracksToMp4(silent, shortAudio, output); + const verified = await verifyRemuxedRecording(silent, shortAudio, output, { + requireAudio: true, + }); + expect(verified.sourcePreserved).toBe(true); + }); + test("rejects shifted audio despite identical encoded content", async () => { + const output = join(directory, "proof-shifted-audio.mp4"); + await run([ + "ffmpeg", + "-v", + "error", + "-i", + silent, + "-itsoffset", + "0.5", + "-i", + silent, + "-map", + "0:v:0", + "-map", + "1:a:0", + "-c", + "copy", + output, + ]); + await expect( + proveRecordingPackets(silent, silent, output, AbortSignal.timeout(5000)), + ).rejects.toThrow(); + await expect( + verifyRemuxedRecording(silent, silent, output, { requireAudio: true }), + ).rejects.toThrow(); + }); + test("does not start cancelled packet verification", async () => { + const controller = new AbortController(); + controller.abort(); + await expect( + verifyRemuxedRecording(silent, silent, silent, { + requireAudio: true, + abortSignal: controller.signal, + }), + ).rejects.toThrow(); + }); +}); + describe("complete recording decode", () => { test.each([ { audio: true, hasAudio: true }, @@ -640,6 +885,7 @@ describe("complete recording decode", () => { requireAudio: false, timeoutMs: 110_000, }); + const stockElapsed = performance.now() - stockStarted; expect(stock.integrity?.video.contentSha256).toBe( source.integrity.video.contentSha256, ); @@ -648,8 +894,25 @@ describe("complete recording decode", () => { ); expect(source.video).toEqual(stock.video); expect(source.audio).toEqual(stock.audio); + const output = join(directory, "long-recording-remux.mp4"); + await muxMediaTracksToMp4(input, input, output); + const efficientStarted = performance.now(); + const efficient = await verifyRemuxedRecording(input, input, output, { + requireAudio: true, + timeoutMs: 30_000, + }); + const efficientElapsed = performance.now() - efficientStarted; + expect(efficient.sourcePreserved).toBe(true); + expect(efficient.integrity).toBeUndefined(); + expect(efficient.video).toEqual(source.video); + expect(efficient.audio).toEqual(source.audio); + expect(efficientElapsed).toBeLessThan(30_000); console.info( - `Recording decode: ${elapsed.toFixed(0)} ms, stock ${(performance.now() - stockStarted).toFixed(0)} ms, ${((sourcePeak - baseline) / 1_024 / 1_024).toFixed(1)} MiB peak RSS increase`, + `Packet-bound full verification: ${efficientElapsed.toFixed(0)} ms`, + ); + + console.info( + `Recording decode: ${elapsed.toFixed(0)} ms, stock ${stockElapsed.toFixed(0)} ms, ${((sourcePeak - baseline) / 1_024 / 1_024).toFixed(1)} MiB peak RSS increase`, ); } finally { clearInterval(memory); @@ -1645,7 +1908,10 @@ async function decoderPids(input: string): Promise { processes.map(async (pid) => { try { const command = await readFile(`/proc/${pid}/cmdline`, "utf8"); - return command.includes("ffmpeg") && command.includes(input) + return (command.includes("ffmpeg") || + command.includes("ffprobe") || + command.includes("recording-audio-timing.ts")) && + command.includes(input) ? Number(pid) : null; } catch (error) { @@ -1664,7 +1930,13 @@ async function decoderPids(input: string): Promise { const output = await run(["ps", "-axo", "pid=,command="]); return output .split("\n") - .filter((line) => line.includes("ffmpeg") && line.includes(input)) + .filter( + (line) => + (line.includes("ffmpeg") || + line.includes("ffprobe") || + line.includes("recording-audio-timing.ts")) && + line.includes(input), + ) .map((line) => Number.parseInt(line.trim(), 10)); } @@ -2153,6 +2425,68 @@ function objectResponse( } describe("remote recording object identity", () => { + test("resumes interrupted byte verification without rereading the verified prefix", async () => { + const identity = '"resume-object"'; + let reads = 0; + let sent = 0; + const server = Bun.serve({ + hostname: "127.0.0.1", + port: 0, + fetch(request) { + if (request.method === "HEAD") + return new Response(null, { + headers: { + ETag: identity, + "Content-Length": String(silentBytes.length), + }, + }); + const range = request.headers.get("range"); + if (range === "bytes=0-0") + return new Response(silentBytes.slice(0, 1), { + status: 206, + headers: { + ETag: identity, + "Content-Range": `bytes 0-0/${silentBytes.length}`, + }, + }); + expect(request.headers.get("if-match")).toBe(identity); + reads++; + if (reads === 1) { + sent += 128; + return new Response(silentBytes.slice(0, 128), { + headers: { ETag: identity }, + }); + } + expect(range).toBe(`bytes=128-${silentBytes.length - 1}`); + sent += silentBytes.length - 128; + return new Response(silentBytes.slice(128), { + status: 206, + headers: { + ETag: identity, + "Content-Range": `bytes 128-${silentBytes.length - 1}/${silentBytes.length}`, + }, + }); + }, + }); + try { + const result = await verifyRemoteRecordingBytes( + `http://127.0.0.1:${server.port}/recording.mp4`, + { + expectedObjectIdentity: identity, + expectedSha256: createHash("sha256") + .update(silentBytes) + .digest("hex"), + expectedFileSize: silentBytes.length, + }, + ); + expect(result.fileSize).toBe(silentBytes.length); + expect(reads).toBe(2); + expect(sent).toBe(silentBytes.length); + } finally { + await server.stop(true); + } + }); + test("binds remote bytes without manufacturing decoded evidence", async () => { const identity = '"byte-bound-output"'; const sha256 = createHash("sha256").update(silentBytes).digest("hex"); diff --git a/apps/media-server/src/lib/media-transfer.ts b/apps/media-server/src/lib/media-transfer.ts index c3c3e58774..6272077939 100644 --- a/apps/media-server/src/lib/media-transfer.ts +++ b/apps/media-server/src/lib/media-transfer.ts @@ -219,15 +219,32 @@ export async function downloadDriveRevision( for (let attempt = 0; attempt < 3 && bytes < target.size; attempt++) { options.signal?.throwIfAborted(); const start = bytes; - const response = await fetcher(target.url, { - headers: { - Authorization: target.authorization, - "Accept-Encoding": "identity", - ...(start ? { Range: `bytes=${start}-` } : {}), - }, - signal: options.signal, - redirect: "error", - }); + let response: Response; + try { + response = await fetcher(target.url, { + headers: { + Authorization: target.authorization, + "Accept-Encoding": "identity", + ...(start ? { Range: `bytes=${start}-` } : {}), + }, + signal: options.signal, + redirect: "error", + }); + } catch (error) { + options.signal?.throwIfAborted(); + if (error instanceof MediaTransferBudgetError || attempt === 2) + throw error; + await Bun.sleep(250 * 2 ** attempt); + continue; + } + if ( + [408, 429, 500, 502, 503, 504].includes(response.status) && + attempt < 2 + ) { + await response.body?.cancel(); + await Bun.sleep(250 * 2 ** attempt); + continue; + } const valid = start ? response.status === 206 && response.headers.get("content-range") === diff --git a/apps/media-server/src/lib/media-video.ts b/apps/media-server/src/lib/media-video.ts index a43b9a4b36..d6737224e2 100644 --- a/apps/media-server/src/lib/media-video.ts +++ b/apps/media-server/src/lib/media-video.ts @@ -14,6 +14,7 @@ import { } from "./media-common"; import { probeVideoFile } from "./media-probe"; import { fetchMedia, materializeMedia } from "./media-transfer"; +import { readRecordingAudioTail } from "./recording-packet-proof"; import { RecordingTimingError, readRecordingVideoTiming, @@ -2473,16 +2474,29 @@ export async function muxMediaTracksToMp4( outputPath: string, abortSignal?: AbortSignal, ): Promise { - if (abortSignal?.aborted) throw new Error("Recording mux was cancelled"); + abortSignal?.throwIfAborted(); + abortSignal = AbortSignal.any([ + ...(abortSignal ? [abortSignal] : []), + AbortSignal.timeout(PROCESS_TIMEOUT_MS), + ]); const startedAt = performance.now(); const timing = await readRecordingVideoTiming(videoInputPath, { abortSignal, timeoutMs: PROCESS_TIMEOUT_MS, }); - if (abortSignal?.aborted) throw new Error("Recording mux was cancelled"); + abortSignal?.throwIfAborted(); const lastTimestamp = timing.lastTimestampTicks - timing.firstTimestampTicks; // FFmpeg 7 can discard a fragmented MP4's stored final sample duration. const videoTimingFilter = `setts=pts=PTS:dts=DTS:duration=if(eq(PTS-STARTPTS\\,${lastTimestamp})\\,${timing.lastDurationTicks}\\,DURATION)`; + const audioTiming = audioInputPath + ? await readRecordingAudioTail(audioInputPath, abortSignal, true) + : undefined; + if (audioTiming && audioTiming.packetCount === undefined) + throw new Error("Recording audio packet count is missing"); + // FFmpeg synthesizes nominal AAC durations, so bind the final duration to the stored source sample. + const audioTimingFilter = audioTiming?.packetCount + ? `setts=duration=if(eq(N\\,${audioTiming.packetCount - 1})\\,${audioTiming.durationTicks}\\,DURATION)` + : undefined; const args = audioInputPath ? [ "ffmpeg", @@ -2501,6 +2515,7 @@ export async function muxMediaTracksToMp4( "copy", "-bsf:v", videoTimingFilter, + ...(audioTimingFilter ? ["-bsf:a", audioTimingFilter] : []), "-avoid_negative_ts", "disabled", "-movie_timescale", diff --git a/apps/media-server/src/lib/recording-audio-timing.ts b/apps/media-server/src/lib/recording-audio-timing.ts new file mode 100644 index 0000000000..d6faa2f415 --- /dev/null +++ b/apps/media-server/src/lib/recording-audio-timing.ts @@ -0,0 +1,106 @@ +import { createHash } from "node:crypto"; +import { EncodedPacketSink, FilePathSource, Input, MP4 } from "mediabunny"; +import { RecordingTimingError } from "./recording-timing"; + +async function inspectAudioTail(path: string, countPackets = false) { + const source = new FilePathSource(path).ref(); + const input = new Input({ formats: [MP4], source }); + let formatReady = false; + try { + await input.getFormat(); + formatReady = true; + const tracks = await input.getAudioTracks(); + if (tracks.length !== 1) + throw new Error("Recording audio tracks are ambiguous"); + const sink = new EncodedPacketSink(tracks[0]); + const packet = await sink.getPacket(Number.POSITIVE_INFINITY, { + skipLiveWait: true, + }); + if ( + !packet || + (await sink.getNextPacket(packet, { + metadataOnly: true, + skipLiveWait: true, + })) + ) + throw new Error("Recording audio tail is ambiguous"); + const scale = await tracks[0].getTimeResolution(); + const scaled = packet.duration * scale; + const ticks = Math.round(scaled); + const tolerance = Math.max( + 0.0000001, + Math.abs(scaled) * Number.EPSILON * 4, + ); + if ( + !Number.isSafeInteger(scale) || + scale <= 0 || + !Number.isSafeInteger(ticks) || + ticks <= 0 || + tolerance >= 0.25 || + Math.abs(scaled - ticks) > tolerance || + !Number.isSafeInteger(packet.sequenceNumber) || + packet.sequenceNumber < 0 + ) + throw new Error("Recording audio duration is not exact"); + let packetCount: number | undefined; + if (countPackets) { + packetCount = 0; + for await (const _packet of sink.packets(undefined, undefined, { + metadataOnly: true, + skipLiveWait: true, + })) { + packetCount++; + } + } + if ( + packetCount !== undefined && + (!Number.isSafeInteger(packetCount) || packetCount <= 0) + ) + throw new Error("Recording audio packet count is invalid"); + return { + packetCount, + durationTicks: ticks, + timeScale: scale, + size: packet.byteLength, + hash: `SHA256:${createHash("sha256").update(packet.data).digest("hex")}`, + }; + } catch (error) { + const retryable = + typeof error === "object" && + error !== null && + "code" in error && + typeof error.code === "string"; + const failure = new RecordingTimingError( + retryable + ? "Recording audio timing inspection was interrupted" + : "Recording audio timing is invalid", + retryable, + ); + failure.cause = error; + throw failure; + } finally { + // Mediabunny 1.45 disposal leaks rejections after failed format detection and can strand pending reads. + if (formatReady) input.dispose(); + else source.free(); + } +} + +if (import.meta.main) { + try { + const path = process.argv[2]; + if (!path) throw new Error("Missing audio timing path"); + console.log( + JSON.stringify(await inspectAudioTail(path, process.argv[3] === "count")), + ); + } catch (error) { + console.log( + JSON.stringify({ + error: + error instanceof RecordingTimingError && !error.retryable + ? "invalid" + : "unavailable", + }), + ); + process.exitCode = 1; + } +} diff --git a/apps/media-server/src/lib/recording-packet-proof.ts b/apps/media-server/src/lib/recording-packet-proof.ts new file mode 100644 index 0000000000..59cf380608 --- /dev/null +++ b/apps/media-server/src/lib/recording-packet-proof.ts @@ -0,0 +1,459 @@ +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { lstat } from "node:fs/promises"; +import { isAbsolute } from "node:path"; +import { fileURLToPath } from "node:url"; +import { z } from "zod"; +import { + RecordingTimingError, + readRecordingVideoTiming, +} from "./recording-timing"; +import { registerSubprocess, unregisterSubprocess } from "./subprocess"; + +export class RecordingPacketMismatchError extends Error {} + +interface PacketStream { + index: number; + codec_type: "video" | "audio"; + time_base: string; + [key: string]: unknown; +} + +const STREAM_FIELDS = + "index,start_pts,codec_type,codec_name,profile,level,codec_tag_string,width,height,sample_aspect_ratio,pix_fmt,color_range,color_space,color_transfer,color_primaries,chroma_location,field_order,refs,sample_fmt,sample_rate,channels,channel_layout,time_base,extradata_hash"; + +async function runInspector( + command: string, + args: string[], + signal: AbortSignal, + line: (value: string) => void, +) { + signal.throwIfAborted(); + const child = spawn(command, args, { stdio: ["ignore", "pipe", "pipe"] }); + let error = ""; + child.stderr.setEncoding("utf8"); + child.stderr.on("data", (chunk: string) => { + error = (error + chunk).slice(-8192); + }); + const exited = new Promise((resolve, reject) => { + child.once("error", reject); + child.once("close", (code) => resolve(code ?? -1)); + }); + if (!child.pid) { + await exited; + throw new Error("Recording packet inspector could not start"); + } + const managed = registerSubprocess({ + pid: child.pid, + exited, + get exitCode() { + return child.exitCode; + }, + kill: (signal?: NodeJS.Signals | number) => child.kill(signal), + }); + const stop = () => { + child.kill("SIGKILL"); + }; + signal.addEventListener("abort", stop, { once: true }); + child.stdout.setEncoding("utf8"); + let pending = ""; + try { + if (signal.aborted) stop(); + for await (const chunk of child.stdout) { + if (typeof chunk !== "string") + throw new Error("Recording packet metadata is invalid"); + pending += chunk; + let newline = pending.indexOf("\n"); + while (newline !== -1) { + if (newline > 65536) + throw new Error("Recording packet metadata exceeds its bound"); + signal.throwIfAborted(); + line(pending.slice(0, newline).replace(/\r$/, "")); + pending = pending.slice(newline + 1); + newline = pending.indexOf("\n"); + } + if (pending.length > 65536) + throw new Error("Recording packet metadata exceeds its bound"); + } + if (pending) line(pending); + + const code = await exited; + signal.throwIfAborted(); + return { code, hasDiagnostics: Boolean(error) }; + } finally { + stop(); + signal.removeEventListener("abort", stop); + await exited.catch(() => {}); + unregisterSubprocess(managed); + } +} + +async function probe( + path: string, + args: string[], + signal: AbortSignal, + line: (value: string) => void, +) { + const result = await runInspector( + "ffprobe", + [ + "-v", + "error", + "-err_detect", + "explode", + "-protocol_whitelist", + "file", + ...args, + path, + ], + signal, + line, + ); + if (result.code !== 0 || result.hasDiagnostics) + throw new Error("Recording packet inspection failed"); +} + +function time(value: string | undefined, base: string) { + if (!value || !/^-?\d+$/.test(value)) + throw new Error("Recording packet timestamp is missing"); + const match = /^(\d+)\/(\d+)$/.exec(base); + if (!match || BigInt(match[1]) <= 0n || BigInt(match[2]) <= 0n) + throw new Error("Recording packet timebase is invalid"); + const numerator = BigInt(value) * BigInt(match[1]); + let a = numerator < 0n ? -numerator : numerator; + let b = BigInt(match[2]); + while (b) [a, b] = [b, a % b]; + return `${numerator / a}/${BigInt(match[2]) / a}`; +} + +const positiveInteger = z + .number() + .int() + .positive() + .max(Number.MAX_SAFE_INTEGER); +const audioTailSchema = z.object({ + packetCount: positiveInteger.optional(), + durationTicks: positiveInteger, + timeScale: positiveInteger, + size: positiveInteger, + hash: z.string().regex(/^SHA256:[a-f0-9]{64}$/), +}); + +export async function readRecordingAudioTail( + path: string, + signal: AbortSignal, + countPackets = false, +) { + try { + if (!isAbsolute(path)) + throw new RecordingTimingError( + "Recording audio timing is invalid", + false, + ); + let output = ""; + const result = await runInspector( + process.execPath, + [ + fileURLToPath(new URL("./recording-audio-timing.ts", import.meta.url)), + path, + countPackets ? "count" : "tail", + ], + signal, + (line) => { + output += line; + if (output.length > 65536) + throw new Error("Audio timing metadata exceeds its bound"); + }, + ); + const decoded: unknown = JSON.parse(output); + if ( + result.code === 1 && + typeof decoded === "object" && + decoded !== null && + "error" in decoded && + decoded.error === "invalid" + ) + throw new RecordingTimingError( + "Recording audio timing is invalid", + false, + ); + const parsed = audioTailSchema.safeParse(decoded); + if ( + result.code !== 0 || + result.hasDiagnostics || + !parsed.success || + (countPackets && parsed.data.packetCount === undefined) + ) + throw new Error("Recording audio timing response is invalid"); + return { + ...parsed.data, + duration: time( + String(parsed.data.durationTicks), + `1/${parsed.data.timeScale}`, + ), + }; + } catch (error) { + if (error instanceof RecordingTimingError) throw error; + const failure = new RecordingTimingError( + "Recording audio timing inspection was interrupted", + true, + ); + failure.cause = error; + throw failure; + } +} + +async function readTrack( + path: string, + kind: "video" | "audio", + signal: AbortSignal, +) { + if (!isAbsolute(path) || !(await lstat(path)).isFile()) + throw new Error("Recording packet proof requires local regular files"); + const selection = kind === "video" ? "v" : "a"; + let metadata = ""; + await probe( + path, + [ + "-select_streams", + selection, + "-show_streams", + "-show_data_hash", + "sha256", + "-show_entries", + `stream=${STREAM_FIELDS}:stream_side_data`, + "-of", + "json", + ], + signal, + (line) => { + metadata += line; + if (metadata.length > 65536) + throw new Error("Recording stream metadata exceeds its bound"); + }, + ); + const parsed: unknown = JSON.parse(metadata); + if ( + !parsed || + typeof parsed !== "object" || + !("streams" in parsed) || + !Array.isArray(parsed.streams) || + parsed.streams.length !== 1 + ) + throw new Error("Recording stream selection is ambiguous"); + const stream = parsed.streams[0] as PacketStream; + if ( + stream.codec_type !== kind || + typeof stream.time_base !== "string" || + !Number.isSafeInteger(stream.index) || + typeof stream.extradata_hash !== "string" || + !/^SHA256:[a-f0-9]{64}$/.test(stream.extradata_hash) + ) + throw new Error("Recording codec configuration is incomplete"); + const { index, time_base: base, start_pts: startPts, ...rest } = stream; + const start = time(String(startPts), base); + const configuration = { ...rest, start }; + const [startNumerator, startDenominator] = start.split("/").map(Number); + const startTime = startNumerator / startDenominator; + if (!Number.isFinite(startTime)) + throw new Error("Recording start time is invalid"); + const tail = + kind === "audio" ? await readRecordingAudioTail(path, signal) : undefined; + const hash = createHash("sha256"); + let count = 0; + let pendingPacket: Record | undefined; + const commitPacket = (fields: Record, nextDts?: string) => { + let { + stream_index: _index, + pts, + dts, + duration, + flags, + ...content + } = fields; + const skipSamples = Object.entries(content).find( + ([key]) => key === "skip_samples" || key.endsWith(":skip_samples"), + )?.[1]; + // FFmpeg can omit only the fully skipped first AAC packet's duration in fragmented MP4. + if ( + duration === "N/A" && + count === 0 && + kind === "audio" && + stream.codec_name === "aac" && + pts === dts && + /^-?\d+$/.test(dts ?? "") && + typeof nextDts === "string" && + /^-?\d+$/.test(nextDts) && + /^[1-9]\d*$/.test(skipSamples ?? "") && + typeof stream.sample_rate === "string" && + /^[1-9]\d*$/.test(stream.sample_rate) + ) { + const inferred = String(BigInt(nextDts) - BigInt(dts)); + if (time(skipSamples, `1/${stream.sample_rate}`) === time(inferred, base)) + duration = inferred; + } + // MP4 can mark an AAC priming packet discardable while retaining identical skip-sample metadata. + const packetFlags = + kind === "audio" && + Object.entries(content).some( + ([key, value]) => + (key === "skip_samples" || key.endsWith(":skip_samples")) && + /^\d+$/.test(value) && + typeof stream.sample_rate === "string" && + /^\d+$/.test(stream.sample_rate) && + time(value, `1/${stream.sample_rate}`) === time(duration, base), + ) + ? flags?.replaceAll("D", "_") + : flags; + // FFmpeg 7 can replace a fragmented AAC tail's stored duration with its nominal frame duration. + if ( + tail && + nextDts === undefined && + (String(tail.size) !== content.size || tail.hash !== content.data_hash) + ) + throw new Error( + "Recording audio tail does not match its packet inventory", + ); + // Interior timing is bound by PTS/DTS; stored terminal durations are compared separately. + hash.update( + JSON.stringify({ + n: count++, + pts: time(pts, base), + dts: time(dts, base), + flags: packetFlags, + content, + }), + ); + hash.update("\n"); + }; + await probe( + path, + [ + "-select_streams", + selection, + "-show_packets", + "-show_data_hash", + "sha256", + "-show_entries", + "packet=stream_index,pts,dts,duration,size,flags,data_hash:packet_side_data", + "-of", + "compact", + ], + signal, + (line) => { + if (!line.startsWith("packet|")) + throw new Error("Unexpected recording packet metadata"); + const fields = Object.fromEntries( + line + .slice(7) + .split("|") + .map((field) => { + const separator = field.indexOf("="); + if (separator < 1) + throw new Error("Invalid recording packet metadata"); + return [field.slice(0, separator), field.slice(separator + 1)]; + }), + ); + if ( + Number(fields.stream_index) !== index || + !/^[1-9]\d*$/.test(fields.size ?? "") || + !/^SHA256:[a-f0-9]{64}$/.test(fields.data_hash ?? "") + ) + throw new Error("Recording packet content is incomplete"); + if (pendingPacket) commitPacket(pendingPacket, fields.dts); + pendingPacket = fields; + }, + ); + if (pendingPacket) commitPacket(pendingPacket); + if (!count) throw new Error("Recording stream contains no packets"); + return { configuration, count, packets: hash.digest("hex"), startTime, tail }; +} + +export async function proveRecordingPackets( + videoPath: string, + audioPath: string | null, + outputPath: string, + signal: AbortSignal, +) { + const paths = [ + ...new Set([videoPath, ...(audioPath ? [audioPath] : []), outputPath]), + ]; + const before = await Promise.all( + paths.map((path) => lstat(path, { bigint: true })), + ); + const sourceTiming = await readRecordingVideoTiming(videoPath, { + abortSignal: signal, + timeoutMs: 45 * 60_000, + }); + if (sourceTiming.terminalPacketCount !== 1) + throw new Error("Tied terminal samples require decoded source evidence"); + const sourceVideo = await readTrack(videoPath, "video", signal); + const outputVideo = await readTrack(outputPath, "video", signal); + if (JSON.stringify(sourceVideo) !== JSON.stringify(outputVideo)) + throw new RecordingPacketMismatchError( + "Recording encoded video does not preserve the source", + ); + if (audioPath) { + const { tail: sourceTail, ...sourceAudio } = await readTrack( + audioPath, + "audio", + signal, + ); + const { tail: outputTail, ...outputAudio } = await readTrack( + outputPath, + "audio", + signal, + ); + if (JSON.stringify(sourceAudio) !== JSON.stringify(outputAudio)) + throw new RecordingPacketMismatchError( + "Recording encoded audio does not preserve the source", + ); + // MP4 edit-list rounding can change the reported AAC tail without changing decoded samples. + if (sourceTail?.duration !== outputTail?.duration) + throw new Error("Recording audio tail requires decoded source evidence"); + } + const outputTiming = await readRecordingVideoTiming(outputPath, { + abortSignal: signal, + timeoutMs: 45 * 60_000, + }); + const normalizeTiming = (timing: typeof sourceTiming) => ({ + packets: timing.packetTimelineSha256, + terminalPackets: timing.terminalPacketCount, + terminalContent: timing.terminalPacketSha256, + duration: time(String(timing.lastDurationTicks), `1/${timing.timeScale}`), + }); + if ( + JSON.stringify(normalizeTiming(sourceTiming)) !== + JSON.stringify(normalizeTiming(outputTiming)) + ) + throw new RecordingPacketMismatchError( + "Recording terminal packet timing changed", + ); + const assertUnchanged = async () => { + for (const [index, path] of paths.entries()) { + const after = await lstat(path, { bigint: true }); + const initial = before[index]; + if ( + !after.isFile() || + after.dev !== initial.dev || + after.ino !== initial.ino || + after.size !== initial.size || + after.mtimeNs !== initial.mtimeNs || + after.ctimeNs !== initial.ctimeNs + ) + throw new RecordingTimingError( + "Recording changed during packet verification", + false, + ); + } + signal.throwIfAborted(); + }; + await assertUnchanged(); + return { + hasAudio: Boolean(audioPath), + videoPackets: sourceVideo.count, + videoStartTime: outputVideo.startTime, + outputTiming, + assertUnchanged, + }; +} diff --git a/apps/media-server/src/lib/recording-verification.ts b/apps/media-server/src/lib/recording-verification.ts index 040e62b7a5..9a599fbf01 100644 --- a/apps/media-server/src/lib/recording-verification.ts +++ b/apps/media-server/src/lib/recording-verification.ts @@ -4,8 +4,17 @@ import { createReadStream } from "node:fs"; import { lstat } from "node:fs/promises"; import { isAbsolute } from "node:path"; import { Readable } from "node:stream"; +import { setTimeout as sleep } from "node:timers/promises"; import { PROCESS_TIMEOUT_MS } from "./media-common"; -import { fetchMedia, materializeMedia } from "./media-transfer"; +import { + fetchMedia, + MediaTransferBudgetError, + materializeMedia, +} from "./media-transfer"; +import { + proveRecordingPackets, + RecordingPacketMismatchError, +} from "./recording-packet-proof"; import { RecordingTimingError, type RecordingVideoTiming, @@ -13,6 +22,9 @@ import { } from "./recording-timing"; import { registerSubprocess, unregisterSubprocess } from "./subprocess"; +const MAX_LOCAL_VERIFICATION_MS = 2 * 60 * 60_000; +const DECODER_STALL_MS = 5 * 60_000; + const MAX_OUTPUT_LINE_LENGTH = 16_384; const MAX_ERROR_LENGTH = 2_048; const MAX_STDERR_LENGTH = 64 * 1_024; @@ -43,6 +55,7 @@ export interface RecordingVerificationOptions { allowObservedDuration?: boolean; abortSignal?: AbortSignal; timeoutMs?: number; + onProgress?: (progress: { frames: number; totalFrames?: number }) => void; } export interface DecodedVideoEvidence { @@ -359,6 +372,7 @@ async function readFrameEvidence( stream: AsyncIterable, streams: Map, allowVideoTies: boolean, + onProgress?: () => void, ): Promise { const decoder = new TextDecoder(); let pending = ""; @@ -373,6 +387,7 @@ async function readFrameEvidence( pending = pending.slice(newline + 1); newline = pending.indexOf("\n"); } + onProgress?.(); if (pending.length > MAX_OUTPUT_LINE_LENGTH) { throw new Error("Recording decoder output exceeded its limit"); } @@ -474,7 +489,8 @@ function validateEvidence( options: RecordingVerificationOptions, inspectingSource: boolean, videoTiming: RecordingVideoTiming | undefined, -): RecordingSourceEvidence { + includeContentHashes = true, +): RecordingVerificationResult { const video = streams.get(0); const audio = streams.get(1); if (video?.kind !== "video" || video.frameCount === 0) { @@ -530,14 +546,18 @@ function validateEvidence( format: stream.format.join(";"), }; }; - const result: RecordingSourceEvidence = { + const result: RecordingVerificationResult = { fullDecode: true, video: videoEvidence, audio: audioEvidence, - integrity: { - video: integrity(video), - audio: audioEvidence && audio ? integrity(audio) : null, - }, + ...(includeContentHashes + ? { + integrity: { + video: integrity(video), + audio: audioEvidence && audio ? integrity(audio) : null, + }, + } + : {}), }; if (videoTiming?.terminalPacketCount === 1) { const duration = @@ -553,7 +573,12 @@ function validateEvidence( }; } if (options.sourceEvidence) { - assertSourcePreserved(options.sourceEvidence, result); + if (!result.integrity) + throw new Error("Recording decoded integrity is missing"); + assertSourcePreserved(options.sourceEvidence, { + ...result, + integrity: result.integrity, + }); result.sourcePreserved = true; } return result; @@ -629,12 +654,80 @@ export async function inspectRecordingSources( ) { throw new Error("Recording sources must be local regular MP4 files"); } - return decodeRecording( + const result = await decodeRecording( videoInputPath, { ...options, requireAudio: audioInputPath !== null }, undefined, audioInputPath, ); + if (!result.integrity) + throw new Error("Recording decoded integrity is missing"); + return { ...result, integrity: result.integrity }; +} + +export async function verifyRemuxedRecording( + videoPath: string, + audioPath: string | null, + outputPath: string, + options: Pick< + RecordingVerificationOptions, + "requireAudio" | "abortSignal" | "timeoutMs" | "onProgress" + >, +): Promise { + const signal = AbortSignal.any([ + ...(options.abortSignal ? [options.abortSignal] : []), + AbortSignal.timeout(options.timeoutMs ?? MAX_LOCAL_VERIFICATION_MS), + ]); + options = { ...options, abortSignal: signal }; + let proof: Awaited>; + try { + proof = await proveRecordingPackets( + videoPath, + audioPath, + outputPath, + signal, + ); + if (proof.outputTiming.terminalPacketCount !== 1) + throw new Error("Tied terminal samples require decoded source evidence"); + } catch (error) { + signal.throwIfAborted(); + if (error instanceof RecordingPacketMismatchError) throw error; + const sourceEvidence = await inspectRecordingSources( + videoPath, + audioPath, + options, + ); + const result = await verifyRecording(outputPath, { + ...options, + sourceEvidence, + }); + if (!result.sourcePreserved) + throw new Error("Recording source preservation was not verified"); + return { ...result, sourcePreserved: true }; + } + const result = await decodeRecording( + outputPath, + { ...options, timeoutMs: options.timeoutMs ?? MAX_LOCAL_VERIFICATION_MS }, + undefined, + undefined, + proof, + ); + await proof.assertUnchanged(); + if ( + result.video.frameCount !== proof.videoPackets || + Boolean(result.audio) !== proof.hasAudio + ) { + throw new Error("Recording decoded tracks do not match the encoded source"); + } + return { + ...result, + video: { + ...result.video, + startTime: proof.videoStartTime, + endTime: proof.videoStartTime + result.video.duration, + }, + sourcePreserved: true, + }; } export async function verifyRecording( @@ -649,7 +742,8 @@ async function decodeRecording( options: RecordingVerificationOptions, objectIdentity?: string, sourceAudioInput?: string | null, -): Promise { + packetProof?: Awaited>, +): Promise { const startedAt = performance.now(); const timeoutMs = options.timeoutMs ?? PROCESS_TIMEOUT_MS; if ( @@ -658,9 +752,10 @@ async function decodeRecording( (options.expectedDuration === undefined && !options.sourceEvidence && !options.allowObservedDuration && + !packetProof && sourceAudioInput === undefined) || !positiveNumber(timeoutMs) || - timeoutMs > PROCESS_TIMEOUT_MS + timeoutMs > (packetProof ? MAX_LOCAL_VERIFICATION_MS : PROCESS_TIMEOUT_MS) ) { throw new Error("Invalid recording verification budget or duration"); } @@ -690,7 +785,7 @@ async function decodeRecording( if (options.abortSignal?.aborted) { throw new Error("Recording verification was cancelled"); } - let videoTiming: RecordingVideoTiming | undefined; + let videoTiming: RecordingVideoTiming | undefined = packetProof?.outputTiming; if (sourceAudioInput !== undefined || options.sourceEvidence) { const deadline = new AbortController(); const remainingMs = timeoutMs - (performance.now() - startedAt); @@ -744,14 +839,15 @@ async function decodeRecording( true, ); } - const knownAudio = - sourceAudioInput === undefined + const knownAudio = packetProof + ? packetProof.hasAudio + : sourceAudioInput === undefined ? options.sourceEvidence ? Boolean(options.sourceEvidence.audio) : options.hasAudio : sourceAudioInput !== null; const streamDecodedContent = - process.platform !== "win32" && knownAudio !== undefined; + !packetProof && process.platform !== "win32" && knownAudio !== undefined; if (streamDecodedContent && !Bun.semver.satisfies(Bun.version, ">=1.4.0")) { throw new Error( "Recording content verification requires Bun 1.4.0 or newer; upgrade the media-server runtime to avoid double-closing decoded-content pipes", @@ -770,6 +866,8 @@ async function decodeRecording( "-1", "-c:v", "rawvideo", + // Encoded packet proof binds the real timeline; diagnostic frames need monotonic mux timestamps. + ...(packetProof ? ["-bsf:v", "setts=pts=N:dts=N:duration=1"] : []), "-c:a", "pcm_f64le", "-threads", @@ -860,14 +958,16 @@ async function decodeRecording( ] : []), ] - : [ - ...outputOptions, - "-f", - "streamhash", - "-hash", - "sha256", - "pipe:2", - ]), + : packetProof + ? [] + : [ + ...outputOptions, + "-f", + "streamhash", + "-hash", + "sha256", + "pipe:2", + ]), ], streamDecodedContent ? (hashAudio ? 2 : 1) : 0, ), @@ -895,17 +995,50 @@ async function decodeRecording( stop(); }, decodeBudgetMs); const streams = new Map(); + let lastDecodedFrames = 0; + let lastAdvance = performance.now(); + let lastReport = 0; + const progress = () => { + const count = streams.get(0)?.frameCount ?? 0; + const decodedFrames = [...streams.values()].reduce( + (total, stream) => total + stream.frameCount, + 0, + ); + if (decodedFrames > lastDecodedFrames) { + lastDecodedFrames = decodedFrames; + lastAdvance = performance.now(); + } + if (performance.now() - lastReport >= 1000) { + lastReport = performance.now(); + options.onProgress?.({ + frames: count, + totalFrames: packetProof?.videoPackets, + }); + } + }; + const stall = packetProof + ? setInterval(() => { + if (performance.now() - lastAdvance > DECODER_STALL_MS) { + failure ??= new RecordingVerificationError( + "Recording decoder stopped making progress", + true, + ); + stop(); + } + }, 1000) + : undefined; const frames = readFrameEvidence( proc.stdout, streams, videoTiming !== undefined, + progress, ); const errors = readDecoderErrors(proc.stderr, input); frames.catch(stop); errors.catch(stop); const pipes: Readable[] = []; const hashes: ReturnType[] = []; - let result: RecordingSourceEvidence; + let result: RecordingVerificationResult; try { if (streamDecodedContent) { for (const stream of proc.stdio.slice(3)) { @@ -956,19 +1089,21 @@ async function decodeRecording( } stream.contentSha256 = digest.sha256; } - } else { + } else if (!packetProof) { for (const digest of diagnostics.digests) decodeLine(digest, streams); } result = validateEvidence( streams, options, - sourceAudioInput !== undefined, + sourceAudioInput !== undefined || packetProof !== undefined, videoTiming, + !packetProof, ); } catch (error) { throw failure ?? error; } finally { clearTimeout(timeout); + if (stall) clearInterval(stall); options.abortSignal?.removeEventListener("abort", cancel); stop(); await Promise.allSettled([proc.exited, frames, errors, ...hashes]); @@ -1052,59 +1187,88 @@ async function hashRemoteRecording( fileSize: number, abortSignal: AbortSignal, ): Promise { - let response: Response; - try { - response = await fetchMedia(input, { - headers: { - "If-Match": objectIdentity, - "X-Cap-Recording-Verification": "1", - }, - signal: abortSignal, - }); - } catch { - throw new RecordingVerificationError( - "Recording bytes could not be read", - true, - ); - } - try { - if (response.status !== 200) { - throw new RecordingVerificationError( - `Recording byte verification failed: ${response.status}`, - response.status === 408 || - response.status === 429 || - response.status >= 500, - ); - } - if (response.headers.get("etag") !== objectIdentity || !response.body) { - throw new Error("Recording object changed during byte verification"); - } - const hash = createHash("sha256"); - let bytesRead = 0; + const hash = createHash("sha256"); + let bytesRead = 0; + for (let attempt = 0; ; attempt++) { + abortSignal.throwIfAborted(); + let response: Response | undefined; try { - for await (const chunk of response.body) { - bytesRead += chunk.byteLength; - if (bytesRead > fileSize) { - throw new RecordingVerificationError( - "Recording object size changed during byte verification", - false, - ); + const offset = bytesRead; + try { + response = await fetchMedia(input, { + headers: { + "If-Match": objectIdentity, + "X-Cap-Recording-Verification": "1", + ...(offset > 0 ? { Range: `bytes=${offset}-${fileSize - 1}` } : {}), + }, + signal: abortSignal, + }); + } catch (error) { + if (error instanceof MediaTransferBudgetError) throw error; + throw new RecordingVerificationError( + "Recording bytes could not be read", + true, + ); + } + if (offset > 0 && response.status === 200) + throw new Error( + "Recording object size changed during byte verification: resume range ignored", + ); + if (response.status !== (offset > 0 ? 206 : 200)) { + throw new RecordingVerificationError( + `Recording byte verification failed: ${response.status}`, + response.status === 408 || + response.status === 429 || + response.status >= 500, + ); + } + if (response.headers.get("etag") !== objectIdentity || !response.body) + throw new Error("Recording object changed during byte verification"); + if ( + offset > 0 && + response.headers.get("content-range") !== + `bytes ${offset}-${fileSize - 1}/${fileSize}` + ) + throw new Error( + "Recording byte resume range does not match the verified object", + ); + try { + for await (const chunk of response.body) { + if (bytesRead + chunk.byteLength > fileSize) + throw new RecordingVerificationError( + "Recording object size changed during byte verification", + false, + ); + hash.update(chunk); + bytesRead += chunk.byteLength; } - hash.update(chunk); + } catch (error) { + if (error instanceof RecordingVerificationError) throw error; + throw new RecordingVerificationError( + "Recording bytes could not be read completely", + true, + ); } + if (bytesRead !== fileSize) + throw new RecordingVerificationError( + "Recording object size changed during byte verification", + true, + ); + return hash.digest("hex"); } catch (error) { - if (error instanceof RecordingVerificationError) throw error; - throw new RecordingVerificationError( - "Recording bytes could not be read completely", - true, - ); - } - if (bytesRead !== fileSize) { - throw new Error("Recording object size changed during byte verification"); + abortSignal.throwIfAborted(); + if ( + !(error instanceof RecordingVerificationError) || + !error.retryable || + attempt >= 3 || + bytesRead >= fileSize + ) + throw error; + } finally { + if (!response?.body?.locked) + await response?.body?.cancel().catch(() => {}); } - return hash.digest("hex"); - } finally { - if (!response.body?.locked) await response.body?.cancel().catch(() => {}); + await sleep(250 * 2 ** attempt, undefined, { signal: abortSignal }); } } diff --git a/apps/media-server/src/routes/video.ts b/apps/media-server/src/routes/video.ts index 1f1a00547e..334b3d27be 100644 --- a/apps/media-server/src/routes/video.ts +++ b/apps/media-server/src/routes/video.ts @@ -60,13 +60,13 @@ import { uploadFileToStorage, uploadToS3, } from "../lib/media-video"; +import { RecordingTimingError } from "../lib/recording-timing"; import { hashRecordingFile, - inspectRecordingSources, isRetryableRecordingVerificationError, - verifyRecording, verifyRemoteRecording, verifyRemoteRecordingBytes, + verifyRemuxedRecording, } from "../lib/recording-verification"; import type { TempFileHandle } from "../lib/temp-files"; import { cleanupStaleTempFiles } from "../lib/temp-files"; @@ -1722,7 +1722,7 @@ function classifySourceError(error: unknown): RecordingErrorCode { if (/HTTP error 404|Server returned 404/.test(error.message)) return "source-missing"; if ( - /Decoded recording|Invalid decoded recording|Recording has no decoded|Recording video packets are missing|Recording timing has no video track|does not match the completed local file|Verified recording size/.test( + /Decoded recording|Invalid decoded recording|Recording has no decoded|Recording video packets are missing|Recording audio timing is invalid|Recording timing has no video track|does not match the completed local file|Verified recording size/.test( error.message, ) ) @@ -2620,18 +2620,6 @@ async function muxSegmentsAsync( resources: getSystemResources(), }); - updateJob(jobId, { message: "Checking the original recording..." }); - sendCurrentJobWebhook(jobId); - const sourceEvidence = await withJobHeartbeat(jobId, () => - withMuxMemoryGuard(abortController, () => - inspectRecordingSources(combinedVideoPath, combinedAudioPath, { - abortSignal: abortController.signal, - }), - ), - ).catch((error: unknown) => { - errorCode = classifySourceError(error); - throw error; - }); const requiredAudio = context.requiredAudio ?? Boolean(audioInput); const resultPath = join(workDir, "result.mp4"); errorCode = "output-invalid"; @@ -2646,7 +2634,11 @@ async function muxSegmentsAsync( abortController.signal, ), ), - ); + ).catch((error: unknown) => { + if (error instanceof RecordingTimingError) + errorCode = classifySourceError(error); + throw error; + }); const beforeDecode = await lstat(resultPath, { bigint: true }); if (!beforeDecode.isFile()) throw new Error("Recording verification requires a local regular file"); @@ -2655,15 +2647,37 @@ async function muxSegmentsAsync( message: "Checking the processed recording...", }); sendCurrentJobWebhook(jobId); + const verificationStartedAt = Date.now(); const localVerified = await withJobHeartbeat(jobId, () => withMuxMemoryGuard(abortController, () => - verifyRecording(resultPath, { - requireAudio: requiredAudio, - sourceEvidence, - abortSignal: abortController.signal, - }), + verifyRemuxedRecording( + combinedVideoPath, + combinedAudioPath, + resultPath, + { + requireAudio: requiredAudio, + abortSignal: abortController.signal, + onProgress: ({ frames, totalFrames }) => { + updateJob(jobId, { + progress: totalFrames + ? 70 + Math.min(4, Math.floor((5 * frames) / totalFrames)) + : 70, + message: `Checking recording: ${frames.toLocaleString("en-US")} frames verified...`, + }); + }, + }, + ), ), ); + logVideoEvent("video_mux_verification_complete", { + jobId, + videoId, + durationMs: Date.now() - verificationStartedAt, + method: localVerified.integrity ? "decoded-source" : "encoded-packets", + frames: localVerified.video.frameCount, + recordingDuration: localVerified.video.duration, + resources: getSystemResources(), + }); if (!localVerified.sourcePreserved) throw new Error("Recording source preservation was not verified"); updateJob(jobId, { @@ -2791,9 +2805,9 @@ async function muxSegmentsAsync( manifestSha256, inventorySha256: context.inventorySha256, sourcePreserved: true as const, - videoDuration: sourceEvidence.video.duration, - hasAudio: Boolean(sourceEvidence.audio), - audioVerified: Boolean(sourceEvidence.audio), + videoDuration: localVerified.video.duration, + hasAudio: Boolean(localVerified.audio), + audioVerified: Boolean(localVerified.audio), }, } : {}),