Skip to content

Commit 12b9b97

Browse files
committed
fix: addressed timing race condition
1 parent e605ec1 commit 12b9b97

2 files changed

Lines changed: 15 additions & 0 deletions

File tree

api/src/queue/queue.service.ts

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,14 @@ export class QueueService {
9292
});
9393
}
9494

95+
/** Refresh the lock timestamp for a task still owned by this worker. No-op if the task was reclaimed by another worker or is no longer RUNNING. */
96+
async refreshLock(taskId: string, workerId: string): Promise<void> {
97+
await this.prisma.queueTask.updateMany({
98+
where: { id: taskId, lockedBy: workerId, status: 'RUNNING' },
99+
data: { lockedAt: new Date() },
100+
});
101+
}
102+
95103
/**
96104
* Mark a task failed. Retries with exponential backoff while attempts remain;
97105
* otherwise settles as FAILED. Returns true if it will be retried.

api/src/worker/worker.service.ts

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -78,6 +78,11 @@ export class WorkerService implements OnModuleInit, OnModuleDestroy {
7878

7979
private async process(task: QueueTask): Promise<void> {
8080
this.logger.log(`Processing task ${task.id} (${task.kind}) for job ${task.jobId}`);
81+
const heartbeatMs = Math.floor(this.config.get('QUEUE_LOCK_TTL_MS') / 3);
82+
const heartbeat = setInterval(
83+
() => void this.queue.refreshLock(task.id, WORKER_ID),
84+
heartbeatMs,
85+
);
8186
try {
8287
await this.orchestrator.processTask(task);
8388
await this.queue.complete(task.id);
@@ -91,6 +96,8 @@ export class WorkerService implements OnModuleInit, OnModuleDestroy {
9196
this.logger.error(`failed to escalate exhausted task: ${(err as Error).message}`),
9297
);
9398
}
99+
} finally {
100+
clearInterval(heartbeat);
94101
}
95102
}
96103

0 commit comments

Comments
 (0)