feat(microservices): add topic-consumers option to server-kafka

integration/microservices/e2e/kafka-topic-consumers.spec.ts

@@ -0,0 +1,96 @@
+import { INestApplication } from '@nestjs/common';
+import { MicroserviceOptions, Transport } from '@nestjs/microservices';
+import { Test } from '@nestjs/testing';
+import { expect } from 'chai';
+import * as request from 'supertest';
+import { KafkaTopicConsumersController } from '../src/kafka-topic-consumers/kafka-topic-consumers.controller';
+import { KafkaTopicConsumersMessagesController } from '../src/kafka-topic-consumers/kafka-topic-consumers.messages.controller';
+
+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
+
+/**
+ * Skip this test in CI/CD pipeline as it requires a running Kafka broker.
+ * Run locally with: npm run test:docker:up && npm run test:integration
+ */
+describe.skip('Kafka topicConsumers', function () {
+  let server: any;
+  let app: INestApplication;
+
+  this.timeout(30000);
+
+  before('Start Kafka app with topicConsumers enabled', async () => {
+    const module = await Test.createTestingModule({
+      controllers: [
+        KafkaTopicConsumersController,
+        KafkaTopicConsumersMessagesController,
+      ],
+    }).compile();
+
+    app = module.createNestApplication();
+    server = app.getHttpAdapter().getInstance();
+
+    app.connectMicroservice<MicroserviceOptions>({
+      transport: Transport.KAFKA,
+      options: {
+        client: {
+          brokers: ['localhost:9092'],
+        },
+        topicConsumers: true,
+      },
+    });
+
+    app.enableShutdownHooks();
+    await app.startAllMicroservices();
+    await app.init();
+  });
+
+  beforeEach(async () => {
+    KafkaTopicConsumersMessagesController.TOPIC_A_PROCESSED = false;
+    KafkaTopicConsumersMessagesController.TOPIC_B_PROCESSED = false;
+    // ensure topic-a handler is non-blocking before each test
+    await request(server).post('/release-a').send();
+  });
+
+  it('should process events from both topics', async () => {
+    await Promise.all([
+      request(server).post('/emit-a').send().expect(200),
+      request(server).post('/emit-b').send().expect(200),
+    ]);
+
+    await sleep(3000);
+
+    expect(KafkaTopicConsumersMessagesController.TOPIC_A_PROCESSED).to.be.true;
+    expect(KafkaTopicConsumersMessagesController.TOPIC_B_PROCESSED).to.be.true;
+  });
+
+  it('should process topic-b while topic-a handler is blocking', async () => {
+    // Block topic-a handler
+    await request(server).post('/block-a').send().expect(200);
+
+    // Emit to topic-a — handler will block until released
+    await request(server).post('/emit-a').send().expect(200);
+
+    // Give time for the message to be consumed and the handler to start blocking
+    await sleep(2000);
+
+    // Emit to topic-b — with topicConsumers=true this should be processed
+    // independently, regardless of topic-a being blocked
+    await request(server).post('/emit-b').send().expect(200);
+
+    await sleep(2000);
+
+    // topic-b must be processed despite topic-a still being blocked
+    expect(KafkaTopicConsumersMessagesController.TOPIC_B_PROCESSED).to.be.true;
+    expect(KafkaTopicConsumersMessagesController.TOPIC_A_PROCESSED).to.be.false;
+
+    // Release topic-a and verify it completes
+    await request(server).post('/release-a').send().expect(200);
+    await sleep(2000);
+
+    expect(KafkaTopicConsumersMessagesController.TOPIC_A_PROCESSED).to.be.true;
+  });
+
+  after('Stop Kafka app', async () => {
+    await app.close();
+  });
+});

integration/microservices/src/kafka-topic-consumers/kafka-topic-consumers.controller.ts

@@ -0,0 +1,43 @@
+import {
+  Controller,
+  HttpCode,
+  OnModuleDestroy,
+  OnModuleInit,
+  Post,
+} from '@nestjs/common';
+import { Client, ClientKafka, Transport } from '@nestjs/microservices';
+
+@Controller()
+export class KafkaTopicConsumersController
+  implements OnModuleInit, OnModuleDestroy
+{
+  @Client({
+    transport: Transport.KAFKA,
+    options: {
+      client: {
+        brokers: ['localhost:9092'],
+      },
+    },
+  })
+  private readonly client: ClientKafka;
+
+  async onModuleInit() {
+    await this.client.connect();
+  }
+
+  async onModuleDestroy() {
+    await this.client.close();
+  }
+
+  @Post('emit-a')
+  @HttpCode(200)
+  emitA() {
+    return this.client.emit('topic-consumers.topic-a', {});
+  }
+
+  @Post('emit-b')
+  @HttpCode(200)
+  emitB() {
+    return this.client.emit('topic-consumers.topic-b', {});
+  }
+}

integration/microservices/src/kafka-topic-consumers/kafka-topic-consumers.messages.controller.ts

@@ -0,0 +1,42 @@
+import { Controller, HttpCode, Post } from '@nestjs/common';
+import { EventPattern } from '@nestjs/microservices';
+import { BehaviorSubject } from 'rxjs';
+import { first, skipWhile } from 'rxjs/operators';
+
+@Controller()
+export class KafkaTopicConsumersMessagesController {
+  static TOPIC_A_PROCESSED = false;
+  static TOPIC_B_PROCESSED = false;
+
+  public waiting = new BehaviorSubject<boolean>(false);
+
+  @Post('block-a')
+  @HttpCode(200)
+  blockA() {
+    this.waiting.next(true);
+  }
+
+  @Post('release-a')
+  @HttpCode(200)
+  releaseA() {
+    this.waiting.next(false);
+  }
+
+  @EventPattern('topic-consumers.topic-a')
+  async handleTopicA(): Promise<void> {
+    await new Promise<void>(resolve => {
+      this.waiting
+        .pipe(
+          skipWhile(isWaiting => isWaiting),
+          first(),
+        )
+        .subscribe(() => resolve());
+    });
+    KafkaTopicConsumersMessagesController.TOPIC_A_PROCESSED = true;
+  }
+
+  @EventPattern('topic-consumers.topic-b')
+  handleTopicB(): void {
+    KafkaTopicConsumersMessagesController.TOPIC_B_PROCESSED = true;
+  }
+}

packages/microservices/interfaces/microservice-configuration.interface.ts

@@ -347,5 +347,17 @@ export interface KafkaOptions {
     deserializer?: Deserializer;
     parser?: KafkaParserConfig;
     producerOnlyMode?: boolean;
+    /**
+     * When enabled, creates a separate Kafka consumer per registered topic,
+     * allowing concurrent message processing across topics.
+     * Each consumer uses `{groupId}-{topic}` as its consumer group ID,
+     * providing independent offset tracking per topic.
+     *
+     * Note: topic name is appended as groupId suffix. Ensure topic names
+     * contain only valid Kafka consumer group ID characters (alphanumeric, '.', '_', '-').
+     *
+     * @default false
+     */
+    topicConsumers?: boolean;
   };
 }