kafkamirrormaker 2 failing with connectionexception

[phanindra-rd]

Hello,

I need a help with kafkamirrormaker setup:

this is my setup details:

Both Kafka versions:
kafka:
version: 4.0.0
metadataVersion: "4.0"
KafkaMirrorMaker Version is also 4.0.0

Mirrormaker is connecting to destination cluster and created its own topic except one from the source topic

Status of MirrorMaker2 deployment is:

error from kafkaoperator:
strimzi-cluster-operator-c6584556d-5fn4v java.net.ConnectException: null strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.common.Utils.toConnectException(Utils.java:1107) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.connectAsync(PlainHttpConnection.java:198) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.checkRetryConnect(PlainHttpConnection.java:230) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.lambda$connectAsync$1(PlainHttpConnection.java:206) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:911) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1773) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.lang.Thread.run(Thread.java:840) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v Caused by: java.nio.channels.ClosedChannelException strimzi-cluster-operator-c6584556d-5fn4v at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:195) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at sun.nio.ch.SocketChannelImpl.beginConnect(SocketChannelImpl.java:760) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:848) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.lambda$connectAsync$0(PlainHttpConnection.java:183) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at java.security.AccessController.doPrivileged(AccessController.java:569) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.connectAsync(PlainHttpConnection.java:185) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v ... 9 more strimzi-cluster-operator-c6584556d-5fn4v 2025-12-18 12:47:25 INFO CrdOperator:123 - Reconciliation #46738(timer) KafkaMirrorMaker2(demo-devops/demo-app-mirrormaker): Status of KafkaMirrorMaker2 demo-app-mirrormaker in namespace demo-devops has been updated strimzi-cluster-operator-c6584556d-5fn4v 2025-12-18 12:47:25 INFO AbstractOperator:520 - Reconciliation #46790(watch) KafkaMirrorMaker2(demo-devops/demo-app-mirrormaker): KafkaMirrorMaker2 demo-app-mirrormaker in namespace demo-devops was MODIFIED strimzi-cluster-operator-c6584556d-5fn4v 2025-12-18 12:47:25 WARN AbstractOperator:566 - Reconciliation #46738(timer) KafkaMirrorMaker2(demo-devops/demo-app-mirrormaker): Failed to reconcile strimzi-cluster-operator-c6584556d-5fn4v java.net.ConnectException: null strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.common.Utils.toConnectException(Utils.java:1107) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.connectAsync(PlainHttpConnection.java:198) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.checkRetryConnect(PlainHttpConnection.java:230) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.lambda$connectAsync$1(PlainHttpConnection.java:206) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.CompletableFuture.uniHandle(CompletableFuture.java:934) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.CompletableFuture$UniHandle.tryFire(CompletableFuture.java:911) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1773) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at java.lang.Thread.run(Thread.java:840) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v Caused by: java.nio.channels.ClosedChannelException strimzi-cluster-operator-c6584556d-5fn4v at sun.nio.ch.SocketChannelImpl.ensureOpen(SocketChannelImpl.java:195) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at sun.nio.ch.SocketChannelImpl.beginConnect(SocketChannelImpl.java:760) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at sun.nio.ch.SocketChannelImpl.connect(SocketChannelImpl.java:848) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.lambda$connectAsync$0(PlainHttpConnection.java:183) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v at java.security.AccessController.doPrivileged(AccessController.java:569) ~[?:?] strimzi-cluster-operator-c6584556d-5fn4v at jdk.internal.net.http.PlainHttpConnection.connectAsync(PlainHttpConnection.java:185) ~[java.net.http:?] strimzi-cluster-operator-c6584556d-5fn4v ... 9 more strimzi-cluster-operator-c6584556d-5fn4v 2025-12-18 12:47:25 INFO AbstractOperator:266 - Reconciliation #46790(watch) KafkaMirrorMaker2(demo-devops/demo-app-mirrormaker): KafkaMirrorMaker2 demo-app-mirrormaker will be checked for creation or modification

not sure what is the issue. struggling to understand

[scholzj]

You will need to start by:

• Sharing your configuration (the custom resources)
• Sharing the full logs - at least from the operator(s) and Mirror Maker
• Make sure things are properly formatted and readable (because what you shared now is not readable)

[phanindra-rd]

apiVersion: kafka.strimzi.io/v1beta2
kind: KafkaMirrorMaker2
metadata:
  name: demo-app-mirrormaker
  namespace: demo-devops
spec:
  version: 4.0.0
  connectCluster: "kafka-demo-app-dr"
  resources:
    requests:
      cpu: 100m
      memory: 1Gi
    limits:
      cpu: 500m
      memory: 2Gi
  clusters:
  - alias: "kafka-demo-app-dev"
    bootstrapServers: kafka-demo-app-kafka-bootstrap.demo-devops.svc.k8s-1.dev:9092
    authentication:
      type: scram-sha-512
      username: admin
      passwordSecret:
        password: password
        secretName: admin
  - alias: "kafka-demo-app-dr"
    bootstrapServers: kafka-demo-app.dr.gcp.company.com:9094
    authentication:
      passwordSecret:
        password: password
        secretName: dr-admin
      type: scram-sha-512
      username: dr-admin
    tls:
      trustedCertificates:
        - secretName: kafka-demo-app-cluster-ca-cert
          certificate: ca.crt
        - secretName: kafka-demo-app-truststore-cert
          certificate: dr-ca.crt
        - secretName: kafka-demo-app-truststore-cert
          certificate: tls.crt
    config:
      # for near real-time offset sync (optional but recommended for failover)
      offset.lag.max: 0
      # -1 means it will use the default replication factor configured in the broker
      config.storage.replication.factor: -1
      offset.storage.replication.factor: -1
      status.storage.replication.factor: -1
  mirrors:
  - sourceCluster: "kafka-demo-app-dev" # OnPrem
    targetCluster: "kafka-demo-app-dr" # GKE
    topicsPattern: ".*"
    groupsPattern: ".*"
    sourceConnector:
      tasksMax: 1
      config:
        replication.factor: 3
        replication.policy.class: "org.apache.kafka.connect.mirror.IdentityReplicationPolicy"
        sync.topic.acls.enabled: "false"
        offset-syncs.topic.replication.factor: 3
        sync.topic.configs.enabled: "true"
    heartbeatConnector:
      config:
        heartbeats.topic.replication.factor: 3
    checkpointConnector:
      config:
        checkpoints.topic.replication.factor: 3
        replication.policy.class: "org.apache.kafka.connect.mirror.IdentityReplicationPolicy"
  logging:
    type: inline
    loggers:
      # Set the root logger level
      log4j.rootLogger: "INFO, CONSOLE"
      # Target specific MM2 and Connect loggers
      log4j.logger.org.apache.kafka.connect.mirror: "DEBUG"
      log4j.logger.org.apache.kafka.clients: "WARN"
      # Useful for troubleshooting SCRAM/SASL authentication issues
      log4j.logger.org.apache.kafka.common.security: "DEBUG"

Cluster dev and dr

apiVersion: kafka.strimzi.io/v1beta2
kind: KafkaNodePool
metadata:
  name: kafka-demo-app-pool
  namespace: demo-devops
  labels:
    strimzi.io/cluster: kafka-demo-app
spec:
  replicas: 3
  roles:
    - broker
    - controller
  resources:
    requests:
      memory: 1Gi
      cpu: 500m
    limits:
      memory: 1Gi
      cpu: 2000m
  storage:
    type: jbod
    volumes:
      - id: 0
        type: persistent-claim
        size: 30Gi
        deleteClaim: false
        class: ceph-nbg-1
---
apiVersion: kafka.strimzi.io/v1beta2
kind: Kafka
metadata:
  name: kafka-demo-app
  namespace: demo-devops
  annotations:
    strimzi.io/node-pools: enabled
    strimzi.io/kraft: enabled
spec:
  kafka:
    version: 4.0.0
    metadataVersion: "4.0"
    template:
      pod:
        topologySpreadConstraints:
          - maxSkew: 1
            topologyKey: "kubernetes.io/hostname"
            whenUnsatisfiable: DoNotSchedule
            labelSelector:
              matchLabels:
                strimzi.io/name: kafka-demo-app
        metadata:
          annotations:
            cluster-autoscaler.kubernetes.io/safe-to-evict: "true"
    authorization:
      type: simple
      superUsers:
        - admin
    listeners:
      - name: plain
        port: 9092
        type: internal
        tls: false
        authentication:
          type: scram-sha-512
      - name: external
        port: 9094
        type: loadbalancer
        tls: true
        authentication:
          type: scram-sha-512
        configuration:
          externalTrafficPolicy: Local
          brokerCertChainAndKey:
            secretName: kafka-demo-app-cert
            certificate: tls.crt
            key: tls.key
          bootstrap:
            annotations:
              external-dns.alpha.kubernetes.io/hostname: kafka-demo-app.k8s.dev.company.com
          brokers:
            - broker: 0
              annotations:
                external-dns.alpha.kubernetes.io/hostname: kafka-demo-app-0.k8s.dev.company.com
              advertisedHost: kafka-demo-app-0.k8s.dev.company.com
            - broker: 1
              annotations:
                external-dns.alpha.kubernetes.io/hostname: kafka-demo-app-1.k8s.dev.company.com
              advertisedHost: kafka-demo-app-1.k8s.dev.company.com
            - broker: 2
              annotations:
                external-dns.alpha.kubernetes.io/hostname: kafka-demo-app-2.k8s.dev.company.com
              advertisedHost: kafka-demo-app-2.k8s.dev.company.com
    config:
      offsets.topic.replication.factor: 3
      transaction.state.log.replication.factor: 3
      transaction.state.log.min.isr: 2
      default.replication.factor: 3
      min.insync.replicas: 2
      auto.create.topics.enable: false
      # Increase the time allowed for the initial connection and SASL handshake
      socket.connection.setup.timeout.ms: 30000
      socket.connection.setup.timeout.max.ms: 60000
      # Increase request timeout
      request.timeout.ms: 60000
  entityOperator:
    topicOperator:
      resources:
        requests:
          cpu: 10m
          memory: 128Mi
        limits:
          memory: 512Mi
          cpu: 500m
    userOperator:
      resources:
        requests:
          cpu: 10m
          memory: 128Mi
        limits:
          memory: 512Mi
          cpu: 500m
  kafkaExporter:
    resources:
      requests:
        cpu: 10m
        memory: 64Mi
      limits:
        memory: 128Mi
        cpu: 500m
    topicRegex: ".*"
    groupRegex: ".*"