Skip to content

Commit edd021d

Browse files
authored
Fix partitioned queue worker concurrency tracking (#1272)
## Summary Fixes partitioned queue dispatch so workflows started from persisted queue rows preserve their stored `queuePartitionKey` when they are registered in the worker's in-memory running workflow map. Without this, `workerConcurrency` can be ineffective for partitioned queues: DBOS dequeues workflows for a specific partition, but then starts them without carrying that partition key into `registerRunningWorkflow`. The running workflow is tracked under `queuePartitionKey = undefined`, while the next dequeue for the real partition counts only running workflows with that partition key. As a result, the worker can repeatedly think there are zero running workflows for the partition and mark more workflows `PENDING` on every polling tick. ## Context We discovered this while running a single DBOS worker with a partitioned queue configured with only local worker concurrency: ```ts await DBOS.registerQueue('my-queue', { partitionQueue: true, workerConcurrency: 1000, }); ``` We enqueued a large batch of workflows using the same `queuePartitionKey` and expected at most ~1000 workflows in `PENDING` for that worker/partition. Instead, `PENDING` grew far above the worker concurrency limit. The queue row was correct, the app version was current, and there was only one worker process. The issue appears to be that `executeWorkflowId` loads `wfStatus.queuePartitionKey` from the database, but does not pass it back into the internal workflow params. Later, `internalWorkflow` registers the running workflow using `params.enqueueOptions?.queuePartitionKey`, so the running workflow is counted under `undefined` instead of the actual partition. ## Workaround A workaround is to set global `concurrency` as well as `workerConcurrency`: ```ts await DBOS.registerQueue('my-queue', { partitionQueue: true, workerConcurrency: 1000, concurrency: 1000, }); ``` That works because global `concurrency` is enforced from Postgres state and the SQL filter includes `queue_partition_key`. However, it has additional overhead: DBOS must count currently pending workflows in the database, use stricter transaction behavior, and coordinate through the database even when there is only one worker. We would prefer to avoid that overhead when `workerConcurrency` alone is sufficient. ## Fix This PR preserves `wfStatus.queuePartitionKey` when queue-dispatched or recovered workflows are re-executed from stored workflow status. That ensures `registerRunningWorkflow` records the correct partition key and `countRunningWorkflowsForQueue(queue, partitionKey)` enforces `workerConcurrency` per partition as intended. Also adds a regression test for a partitioned queue with `workerConcurrency: 1` and no global `concurrency`. The first workflow blocks across multiple polling ticks, and the test verifies that no second workflow in the same partition starts until capacity is available. ## Tests ```bash ./node_modules/.bin/jest tests/wfqueue.test.ts --runInBand --testNamePattern="partitioned-queue-worker-concurrency" npm run build ```
1 parent c612cfe commit edd021d

2 files changed

Lines changed: 55 additions & 1 deletion

File tree

src/dbos-executor.ts

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1148,6 +1148,8 @@ export class DBOSExecutor {
11481148

11491149
// If starting a new workflow, assign a new UUID. Otherwise, use the workflow's original UUID.
11501150
const workflowStartID = !!options?.startNewWorkflow ? undefined : workflowID;
1151+
const enqueueOptions =
1152+
wfStatus.queuePartitionKey !== undefined ? { queuePartitionKey: wfStatus.queuePartitionKey } : undefined;
11511153

11521154
if (methReg?.workflowConfig) {
11531155
return await runWithTopContext(recoverCtx, async () => {
@@ -1157,6 +1159,7 @@ export class DBOSExecutor {
11571159
workflowUUID: workflowStartID,
11581160
configuredInstance: configuredInst,
11591161
queueName: wfStatus.queueName,
1162+
enqueueOptions,
11601163
executeWorkflow: true,
11611164
deadlineEpochMS: wfStatus.deadlineEpochMS,
11621165
isRecoveryDispatch: !!options?.isRecoveryDispatch,
@@ -1189,6 +1192,7 @@ export class DBOSExecutor {
11891192
workflowUUID: workflowStartID,
11901193
configuredInstance: configuredInst,
11911194
queueName: wfStatus.queueName, // Probably null
1195+
enqueueOptions,
11921196
executeWorkflow: true,
11931197
isRecoveryDispatch: !!options?.isRecoveryDispatch,
11941198
isQueueDispatch: !!options?.isQueueDispatch,
@@ -1228,6 +1232,7 @@ export class DBOSExecutor {
12281232
tempWfType: TempWorkflowType.send,
12291233
workflowUUID: workflowStartID,
12301234
queueName: wfStatus.queueName,
1235+
enqueueOptions,
12311236
executeWorkflow: true,
12321237
isRecoveryDispatch: !!options?.isRecoveryDispatch,
12331238
isQueueDispatch: !!options?.isQueueDispatch,

tests/wfqueue.test.ts

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1565,7 +1565,7 @@ describe('queue-time-outs', () => {
15651565

15661566
beforeEach(async () => {
15671567
await DBOS.launch();
1568-
for (const ref of [timeoutQueue, dedupRecoveryQueue, partitionQueue]) {
1568+
for (const ref of [timeoutQueue, dedupRecoveryQueue, partitionQueue, partitionWorkerConcurrencyQueue]) {
15691569
await DBOS.registerQueue(ref.name, { onConflict: 'always_update', ...ref.config });
15701570
}
15711571
});
@@ -1792,6 +1792,10 @@ describe('queue-time-outs', () => {
17921792
name: 'partition-queue',
17931793
config: { partitionQueue: true, concurrency: 1, ...testPolling },
17941794
};
1795+
const partitionWorkerConcurrencyQueue: QueueRef = {
1796+
name: 'partition-worker-concurrency-queue',
1797+
config: { partitionQueue: true, workerConcurrency: 1, ...testPolling },
1798+
};
17951799

17961800
const partitionBlockedWorkflow = DBOS.registerWorkflow(
17971801
async () => {
@@ -1809,6 +1813,19 @@ describe('queue-time-outs', () => {
18091813
{ name: 'partitionNormalWorkflow' },
18101814
);
18111815

1816+
const partitionWorkerConcurrencyBlockingEvent = new Event();
1817+
const partitionWorkerConcurrencyStartedEvent = new Event();
1818+
let partitionWorkerConcurrencyStartedCount = 0;
1819+
const partitionWorkerConcurrencyWorkflow = DBOS.registerWorkflow(
1820+
async () => {
1821+
partitionWorkerConcurrencyStartedCount++;
1822+
partitionWorkerConcurrencyStartedEvent.set();
1823+
await partitionWorkerConcurrencyBlockingEvent.wait();
1824+
return DBOS.workflowID;
1825+
},
1826+
{ name: 'partitionWorkerConcurrencyWorkflow' },
1827+
);
1828+
18121829
test('partitioned-queues', async () => {
18131830
const blockedPartitionKey = 'blocked';
18141831
const normalPartitionKey = 'normal';
@@ -1871,6 +1888,38 @@ describe('queue-time-outs', () => {
18711888
}, Error);
18721889
});
18731890

1891+
test('partitioned-queue-worker-concurrency', async () => {
1892+
partitionWorkerConcurrencyBlockingEvent.clear();
1893+
partitionWorkerConcurrencyStartedEvent.clear();
1894+
partitionWorkerConcurrencyStartedCount = 0;
1895+
1896+
const partitionKey = 'same-partition';
1897+
const handles = await Promise.all(
1898+
Array.from({ length: 3 }, () =>
1899+
DBOS.startWorkflow(partitionWorkerConcurrencyWorkflow, {
1900+
queueName: partitionWorkerConcurrencyQueue.name,
1901+
enqueueOptions: { queuePartitionKey: partitionKey },
1902+
})(),
1903+
),
1904+
);
1905+
1906+
try {
1907+
await partitionWorkerConcurrencyStartedEvent.wait();
1908+
await sleepms(500);
1909+
expect(partitionWorkerConcurrencyStartedCount).toBe(1);
1910+
const statuses = await Promise.all(handles.map((h) => h.getStatus()));
1911+
expect(statuses.filter((s) => s?.status === StatusString.PENDING)).toHaveLength(1);
1912+
expect(statuses.filter((s) => s?.status === StatusString.ENQUEUED)).toHaveLength(2);
1913+
for (const status of statuses) {
1914+
expect(status?.queuePartitionKey).toBe(partitionKey);
1915+
}
1916+
} finally {
1917+
partitionWorkerConcurrencyBlockingEvent.set();
1918+
}
1919+
1920+
await Promise.all(handles.map((h) => h.getResult()));
1921+
});
1922+
18741923
test('explicit-queue-listen-test', async () => {
18751924
await DBOS.shutdown();
18761925
const config = generateDBOSTestConfig();

0 commit comments

Comments
 (0)