fix: validate translated transcript speaker annotations

Author: pavzagor

apps/web/__tests__/unit/transcript-translation.test.ts

@@ -0,0 +1,45 @@
+import { describe, expect, it } from "vitest";
+import { isValidTranscriptTranslation } from "@/lib/transcript-vtt";
+
+const source =
+	"WEBVTT\n\n1\n00:00:00.125 --> 00:00:01.500\n<v Speaker A>Hello there.</v>\n\n2\n00:00:01.600 --> 00:00:03.000\n<v Speaker B>Good morning.</v>\n";
+const translated = source
+	.replace("Hello there.", "Merhaba.")
+	.replace("Good morning.", "Günaydın.");
+
+describe("translation structure validation", () => {
+	it("accepts translated speech with unchanged voices and timings", () => {
+		expect(isValidTranscriptTranslation(source, translated)).toBe(true);
+		expect(
+			isValidTranscriptTranslation(source, translated.replace(/\n/g, "\r\n")),
+		).toBe(true);
+		expect(
+			isValidTranscriptTranslation(
+				source,
+				translated.replace("Merhaba.", "Merhaba\narkadaşım."),
+			),
+		).toBe(true);
+	});
+	it.each([
+		translated.replace("<v Speaker A>", ""),
+		translated.replace("<v Speaker A>", "<v Speaker B>"),
+		translated.replace("<v Speaker A>", "<v Konuşmacı A>"),
+		translated.replace("</v>", ""),
+		translated.replace("Merhaba.", "Merhaba.<v Speaker B>Ek söz.</v>"),
+		translated.replace("00:00:00.125", "00:00:00.000"),
+		translated.replace("\n2\n", "\n3\n"),
+		translated.split("\n\n2")[0] ?? "",
+		translated.replace("Merhaba.", ""),
+		`Here is the WEBVTT:\n${translated}`,
+		`${translated}\nExtra explanation`,
+	])("rejects changed metadata or malformed cues before caching", (value) => {
+		expect(isValidTranscriptTranslation(source, value)).toBe(false);
+	});
+	it("accepts legacy captions but rejects invented speakers", () => {
+		const legacy = source.replace(/<[^>]*>/g, "");
+		expect(
+			isValidTranscriptTranslation(legacy, legacy.replace("Hello", "Merhaba")),
+		).toBe(true);
+		expect(isValidTranscriptTranslation(legacy, translated)).toBe(false);
+	});
+});

apps/web/actions/videos/translate-transcript.ts

@@ -12,6 +12,7 @@ import { runWithAiProviders } from "@/lib/ai/run";
 import { isRateLimited, RATE_LIMIT_IDS } from "@/lib/rate-limit";
 import * as EffectRuntime from "@/lib/server";
 import { runPromise } from "@/lib/server";
+import { isValidTranscriptTranslation } from "@/lib/transcript-vtt";
 import { decodeStorageVideo } from "@/lib/video-storage";
 import {
 	type LanguageCode,
@@ -73,6 +74,19 @@ export async function translateTranscript(
 
 	const { video } = query[0];
 
+	const originalVtt = await Effect.gen(function* () {
+		const [bucket] = yield* Storage.getAccessForVideo(
+			decodeStorageVideo(video),
+		);
+		return yield* bucket.getObject(
+			`${video.ownerId}/${videoId}/transcription.vtt`,
+		);
+	}).pipe(runPromise);
+
+	if (Option.isNone(originalVtt)) {
+		return { success: false, message: "Original transcript not found" };
+	}
+
 	const translatedKey = `${video.ownerId}/${videoId}/transcription.${targetLanguage}.vtt`;
 
 	try {
@@ -83,7 +97,10 @@ export async function translateTranscript(
 			return yield* bucket.getObject(translatedKey);
 		}).pipe(runPromise);
 
-		if (Option.isSome(existingTranslation)) {
+		if (
+			Option.isSome(existingTranslation) &&
+			isValidTranscriptTranslation(originalVtt.value, existingTranslation.value)
+		) {
 			return {
 				success: true,
 				translatedVtt: existingTranslation.value,
@@ -94,19 +111,6 @@ export async function translateTranscript(
 		console.debug("[translateTranscript] No cached translation found:", e);
 	}
 
-	const originalVtt = await Effect.gen(function* () {
-		const [bucket] = yield* Storage.getAccessForVideo(
-			decodeStorageVideo(video),
-		);
-		return yield* bucket.getObject(
-			`${video.ownerId}/${videoId}/transcription.vtt`,
-		);
-	}).pipe(runPromise);
-
-	if (Option.isNone(originalVtt)) {
-		return { success: false, message: "Original transcript not found" };
-	}
-
 	const translatedVtt = await translateVttContent(
 		originalVtt.value,
 		targetLanguage,
@@ -167,10 +171,12 @@ ${vttContent}`;
 				...(selection.supportsTemperature ? { temperature: 0.3 } : {}),
 			});
 
-			// Validate inside the provider loop so a fulfilled response that
-			// dropped the WEBVTT header falls through to the next provider.
-			if (!response.text.includes("WEBVTT")) {
-				throw new Error("translation response did not contain WEBVTT");
+			// Validate inside the provider loop so malformed translations try the
+			// next provider without poisoning the cached captions.
+			if (!isValidTranscriptTranslation(vttContent, response.text)) {
+				throw new Error(
+					"translation changed cue structure or speaker annotations",
+				);
 			}
 
 			return response.text.trim();

apps/web/lib/transcript-vtt.ts

@@ -244,3 +244,45 @@ export const parseVTT = (vttContent: string): TranscriptEntry[] => {
 	const sortedEntries = entries.sort((a, b) => a.startTime - b.startTime);
 	return sortedEntries;
 };
+
+function getTranslationCueStructure(content: string) {
+	const blocks = content
+		.trim()
+		.replace(/\r\n?/g, "\n")
+		.split(/\n[\t ]*\n/);
+	if (blocks.shift() !== "WEBVTT" || blocks.length === 0) return null;
+	const cues: { id: string; timing: string; voices: string[] }[] = [];
+	for (const block of blocks) {
+		const [id, timing, ...payload] = block.split("\n");
+		if (
+			!id ||
+			!/^\d+$/.test(id) ||
+			!timing ||
+			!/^\d{2,}:\d{2}:\d{2}\.\d{3} --> \d{2,}:\d{2}:\d{2}\.\d{3}(?:[ \t].*)?$/.test(
+				timing,
+			)
+		)
+			return null;
+		const text = payload.join("\n");
+		if (!parseVttCueText(text).text) return null;
+		cues.push({
+			id,
+			timing,
+			voices: text.match(/<\/?v(?:[.\s][^>]*)?>/g) ?? [],
+		});
+	}
+	return cues;
+}
+
+export function isValidTranscriptTranslation(
+	source: string,
+	translated: string,
+): boolean {
+	const sourceCues = getTranslationCueStructure(source);
+	const translatedCues = getTranslationCueStructure(translated);
+	return (
+		sourceCues !== null &&
+		translatedCues !== null &&
+		JSON.stringify(sourceCues) === JSON.stringify(translatedCues)
+	);
+}