⚡️ Bump up Kafka concurrency

acco · acco · commit 41dc2d4a8652 · 2025-03-04T18:26:50.000-08:00
diff --git a/lib/sequin/runtime/kafka_pipeline.ex b/lib/sequin/runtime/kafka_pipeline.ex
@@ -19,7 +19,7 @@ defmodule Sequin.Runtime.KafkaPipeline do
   def processors_config(%SinkConsumer{}) do
     [
       default: [
-        concurrency: 100,
+        concurrency: System.schedulers_online() * 2,
         max_demand: 10,
         min_demand: 5
       ]
@@ -30,7 +30,7 @@ defmodule Sequin.Runtime.KafkaPipeline do
   def batchers_config(%SinkConsumer{batch_size: batch_size}) do
     [
       default: [
-        concurrency: 20,
+        concurrency: 80,
         batch_size: batch_size,
         batch_timeout: 50
       ]

Original file line number	Diff line number	Diff line change
`@@ -19,7 +19,7 @@ defmodule Sequin.Runtime.KafkaPipeline do`
`19`	`19`	`def processors_config(%SinkConsumer{}) do`
`20`	`20`	`[`
`21`	`21`	`default: [`
`22`		`- concurrency: 100,`
	`22`	`+ concurrency: System.schedulers_online() * 2,`
`23`	`23`	`max_demand: 10,`
`24`	`24`	`min_demand: 5`
`25`	`25`	`]`
`@@ -30,7 +30,7 @@ defmodule Sequin.Runtime.KafkaPipeline do`
`30`	`30`	`def batchers_config(%SinkConsumer{batch_size: batch_size}) do`
`31`	`31`	`[`
`32`	`32`	`default: [`
`33`		`- concurrency: 20,`
	`33`	`+ concurrency: 80,`
`34`	`34`	`batch_size: batch_size,`
`35`	`35`	`batch_timeout: 50`
`36`	`36`	`]`