Skip to content

Commit ecc02f5

Browse files
committed
fix: restrict automatic recovery to actionable recording sources
1 parent 8454b6b commit ecc02f5

4 files changed

Lines changed: 90 additions & 9 deletions

File tree

apps/web/__tests__/unit/desktop-recording-jobs.test.ts

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -895,6 +895,29 @@ describe("retained-source retry policy", () => {
895895
});
896896

897897
describe("recovery admission", () => {
898+
it("keeps an inspected missing source paused until a completion request resumes it", async () => {
899+
const attempt = await createAttempt();
900+
const paused = {
901+
...attempt,
902+
state: "source-blocked" as const,
903+
source: null,
904+
leaseExpiresAt: null,
905+
errorCode: "source-reupload-required",
906+
nextRetryAt: now,
907+
};
908+
rows.jobs = [paused];
909+
expect(isDesktopRecordingJobRecoverable(paused, now)).toBe(false);
910+
expect(await listRecoverableSegmentJobs({ now })).toEqual([]);
911+
const resumed = await ensureSegmentProcessingJob({
912+
videoId,
913+
userId,
914+
verification,
915+
now,
916+
});
917+
expect(resumed.job).toMatchObject({ state: "committing", errorCode: null });
918+
expect(isDesktopRecordingJobRecoverable(resumed.job, now)).toBe(true);
919+
});
920+
898921
it("prioritizes interrupted new recordings over an older missing-source backlog", async () => {
899922
await createAttempt();
900923
const current = {
@@ -928,6 +951,7 @@ describe("recovery admission", () => {
928951
};
929952
rows.jobs = [
930953
"processing-retry-exhausted",
954+
"source-reupload-required",
931955
"output-replaced",
932956
"video-deleting",
933957
].map((errorCode) => ({ ...current, videoId: errorCode, errorCode }));

apps/web/__tests__/unit/desktop-segments-recovery.test.ts

Lines changed: 37 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ vi.mock("@cap/database/schema", () => ({
1616
videoId: "upload.videoId",
1717
phase: "upload.phase",
1818
updatedAt: "upload.updatedAt",
19+
startedAt: "upload.startedAt",
1920
},
2021
videoProcessingJobs: { videoId: "job.videoId" },
2122
}));
@@ -26,6 +27,7 @@ vi.mock("drizzle-orm", () => ({
2627
inArray: (left: unknown, right: unknown) => ({ in: [left, right] }),
2728
isNull: (value: unknown) => ({ null: value }),
2829
lte: (left: unknown, right: unknown) => ({ lte: [left, right] }),
30+
gte: (left: unknown, right: unknown) => ({ gte: [left, right] }),
2931
sql: (strings: TemplateStringsArray, ...values: unknown[]) => ({
3032
strings: [...strings],
3133
values,
@@ -159,6 +161,38 @@ describe("committed source recovery", () => {
159161
});
160162

161163
describe("durable recovery scheduling", () => {
164+
it("does not create a durable job for an unfinished legacy upload", async () => {
165+
const now = new Date("2026-09-09T20:00:00Z");
166+
const legacy = selectChain([{ videoId, ownerId: userId }]);
167+
mocks.db.mockReturnValueOnce(legacy).mockReturnValue(selectChain([video]));
168+
mocks.get.mockReturnValue(
169+
Effect.succeed(
170+
Option.some(JSON.stringify({ ...manifest, is_complete: false })),
171+
),
172+
);
173+
const result = await recoverStaleDesktopSegments({ now });
174+
expect(result.statuses).toEqual({ "source-incomplete": 1 });
175+
expect(mocks.queue).not.toHaveBeenCalled();
176+
expect(mocks.put).not.toHaveBeenCalled();
177+
expect(JSON.stringify(legacy.where.mock.calls)).toContain(
178+
"2026-09-02T20:00:00.000Z",
179+
);
180+
});
181+
182+
it("continues inspecting legacy uploads after a storage failure", async () => {
183+
const legacy = selectChain([
184+
{ videoId, ownerId: userId },
185+
{ videoId: "second", ownerId: userId },
186+
]);
187+
mocks.db.mockReturnValueOnce(legacy).mockReturnValue(selectChain([video]));
188+
mocks.get.mockReturnValueOnce(
189+
Effect.fail(new Error("storage unavailable")),
190+
);
191+
const result = await recoverStaleDesktopSegments();
192+
expect(result.statuses).toEqual({ failed: 1, queued: 1 });
193+
expect(mocks.queue).toHaveBeenCalledTimes(1);
194+
});
195+
162196
it("recovers old retry and expired processing jobs without a recording-age cutoff", async () => {
163197
mocks.db.mockReturnValue(selectChain([]));
164198
mocks.recoverable.mockResolvedValue([
@@ -192,14 +226,14 @@ describe("durable recovery scheduling", () => {
192226

193227
it("adopts stranded legacy processing rows without assuming their inventory is complete", async () => {
194228
const chain = selectChain([{ videoId, ownerId: userId }]);
195-
mocks.db.mockReturnValue(chain);
229+
mocks.db.mockReturnValueOnce(chain).mockReturnValue(selectChain([video]));
196230
mocks.queue.mockRejectedValue(new SourceCommitPendingError());
197231
const result = await recoverStaleDesktopSegments();
198232
expect(result.statuses).toEqual({ "source-committing": 1 });
199233
const query = JSON.stringify(chain.where.mock.calls);
200234
expect(query).toContain('"processing"');
201235
expect(query).not.toContain("28 HOUR");
202-
expect(query).not.toContain("startedAt");
236+
expect(query).toContain("startedAt");
203237
expect(mocks.put).not.toHaveBeenCalled();
204238
});
205239

@@ -231,7 +265,7 @@ describe("durable recovery scheduling", () => {
231265
ownerId: userId,
232266
})),
233267
);
234-
mocks.db.mockReturnValue(legacy);
268+
mocks.db.mockReturnValueOnce(legacy).mockReturnValue(selectChain([video]));
235269
const result = await recoverStaleDesktopSegments();
236270
expect(mocks.recoverable).toHaveBeenCalledWith(
237271
expect.objectContaining({ limit: 15 }),

apps/web/lib/desktop-recording-jobs.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ export const DESKTOP_RECORDING_LEASE_MS = 5 * 60 * 1_000;
3333
export const DESKTOP_RECORDING_SOURCE_RETRY_MS = 60 * 60 * 1_000;
3434
export const DESKTOP_RECORDING_OUTPUT_REPLACED = "output-replaced";
3535
export const DESKTOP_RECORDING_DELETING = "video-deleting";
36+
export const DESKTOP_RECORDING_SOURCE_REUPLOAD_REQUIRED =
37+
"source-reupload-required";
3638
export const DESKTOP_RECORDING_RETRY_EXHAUSTED = "processing-retry-exhausted";
3739
const MAX_AUTOMATIC_ATTEMPTS = 5;
3840
const RETRY_EXHAUSTED_MESSAGE =
@@ -162,6 +164,8 @@ export function isDesktopRecordingJobRecoverable(
162164
now: Date,
163165
) {
164166
if (job.state === "verified") return false;
167+
if (job.errorCode === DESKTOP_RECORDING_SOURCE_REUPLOAD_REQUIRED)
168+
return false;
165169
if (job.errorCode === DESKTOP_RECORDING_RETRY_EXHAUSTED) return false;
166170
if (job.errorCode === DESKTOP_RECORDING_OUTPUT_REPLACED) return false;
167171
if (job.errorCode === DESKTOP_RECORDING_DELETING) return false;
@@ -972,6 +976,10 @@ export async function listRecoverableSegmentJobs({
972976
DESKTOP_RECORDING_OUTPUT_REPLACED,
973977
),
974978
ne(videoProcessingJobs.errorCode, DESKTOP_RECORDING_DELETING),
979+
ne(
980+
videoProcessingJobs.errorCode,
981+
DESKTOP_RECORDING_SOURCE_REUPLOAD_REQUIRED,
982+
),
975983
ne(
976984
videoProcessingJobs.errorCode,
977985
DESKTOP_RECORDING_RETRY_EXHAUSTED,

apps/web/lib/desktop-segments-recovery.ts

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ import {
66
} from "@cap/database/schema";
77
import { Storage } from "@cap/web-backend";
88
import { type User, Video } from "@cap/web-domain";
9-
import { and, asc, eq, inArray, isNull, lte, sql } from "drizzle-orm";
9+
import { and, asc, eq, gte, inArray, isNull, lte, sql } from "drizzle-orm";
1010
import { Effect, Option, Schema } from "effect";
1111
import {
1212
DesktopRecordingSourceBlockedError,
@@ -23,6 +23,8 @@ import { decodeStorageVideo } from "@/lib/video-storage";
2323

2424
export const DESKTOP_SEGMENTS_RECOVERY_MIN_AGE_MS = 60 * 60 * 1_000;
2525
export const DESKTOP_SEGMENTS_RECOVERY_BATCH_SIZE = 20;
26+
export const DESKTOP_SEGMENTS_LEGACY_RECOVERY_MAX_AGE_MS =
27+
7 * 24 * 60 * 60 * 1_000;
2628

2729
const RECOVERABLE_UPLOAD_PHASES = [
2830
"uploading",
@@ -244,20 +246,33 @@ export async function recoverStaleDesktopSegments({
244246
and(
245247
inArray(videoUploads.phase, RECOVERABLE_UPLOAD_PHASES),
246248
lte(videoUploads.updatedAt, staleBefore),
249+
gte(
250+
videoUploads.startedAt,
251+
new Date(now.getTime() - DESKTOP_SEGMENTS_LEGACY_RECOVERY_MAX_AGE_MS),
252+
),
247253
isNull(videoProcessingJobs.videoId),
248254
sql`JSON_UNQUOTE(JSON_EXTRACT(${videos.source}, '$.type')) = 'desktopSegments'`,
249255
),
250256
)
251257
.orderBy(asc(videoUploads.updatedAt), asc(videoUploads.videoId))
252258
.limit(remaining);
253259
for (const candidate of legacy) {
254-
record(
255-
candidate.videoId,
256-
await recoverRecording({
260+
try {
261+
const result = await completeDesktopSegmentsManifestAndQueue({
257262
videoId: candidate.videoId,
258263
userId: candidate.ownerId,
259-
}),
260-
);
264+
});
265+
record(candidate.videoId, result.status);
266+
} catch (error) {
267+
console.error(
268+
"[desktop-segments-recovery] Legacy source inspection failed",
269+
{
270+
videoId: candidate.videoId,
271+
error,
272+
},
273+
);
274+
record(candidate.videoId, "failed");
275+
}
261276
}
262277
return summary;
263278
}

0 commit comments

Comments
 (0)