factorhouse-local/compose-kpow.yml at main · factorhouse/factorhouse-local · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
x-common-environment: &kafka_broker_common_env_vars # KRaft settings
  KAFKA_PROCESS_ROLES: "broker,controller"
  KAFKA_CONTROLLER_LISTENER_NAMES: CONTROLLER
  KAFKA_CONTROLLER_QUORUM_VOTERS: "1@kafka-1:19091,2@kafka-2:19091,3@kafka-3:19091"
  # Replace CLUSTER_ID with a unique base64 UUID using "bin/kafka-storage.sh random-uuid"
  # See https://docs.confluent.io/kafka/operations-tools/kafka-tools.html#kafka-storage-sh
  CLUSTER_ID: ${KAFKA_CLUSTER_ID:-42RglGpZQwy9D5VzTVpCWA}
  # Common Kafka settings
  KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: CONTROLLER:PLAINTEXT,LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT,LISTENER_MINIKUBE:PLAINTEXT
  KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
  KAFKA_CONFLUENT_SUPPORT_METRICS_ENABLE: "false"
  KAFKA_LOG4J_ROOT_LOGLEVEL: INFO
  KAFKA_NUM_PARTITIONS: "3"
  KAFKA_DEFAULT_REPLICATION_FACTOR: "3"
  KAFKA_LOG_CLEANUP_POLICY: "delete"
  KAFKA_LOG_RETENTION_BYTES: "524288000" # 500 MB
  KAFKA_LOG_SEGMENT_BYTES: "67108864" # 64 MB segments
  KAFKA_LOG_RETENTION_MS: "86400000" # 1 day
  # KIP-932 Share Groups configuration
  # Enables the Share Group protocol alongside legacy and new consumer protocols
  KAFKA_GROUP_COORDINATOR_REBALANCE_PROTOCOLS: "classic,consumer,share"
  # Activates the Share Coordinator module on the brokers
  KAFKA_SHARE_COORDINATOR_ENABLE: "true"
  KAFKA_GROUP_SHARE_ENABLE: "true"
  # Required to use preview/early-access APIs in the Kafka 4.0/4.1 lifecycle
  KAFKA_UNSTABLE_API_VERSIONS_ENABLE: "true"

services:
  schema:
    image: confluentinc/cp-schema-registry:8.1.1
    container_name: schema_registry
    ports:
      - "8081:8081"
    networks:
      - factorhouse
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    environment:
      SCHEMA_REGISTRY_HOST_NAME: "schema"
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: "kafka-1:19092,kafka-2:19093,kafka-3:19094"
      SCHEMA_REGISTRY_AUTHENTICATION_METHOD: BASIC
      SCHEMA_REGISTRY_AUTHENTICATION_REALM: schema
      SCHEMA_REGISTRY_AUTHENTICATION_ROLES: schema-admin
      SCHEMA_REGISTRY_OPTS: -Djava.security.auth.login.config=/etc/schema/schema_jaas.conf
    volumes:
      - ./resources/kpow/schema:/etc/schema

  connect:
    image: confluentinc/cp-kafka-connect:8.1.1
    container_name: connect
    restart: unless-stopped
    depends_on:
      - kafka-1
      - kafka-2
      - kafka-3
    ports:
      - 8083:8083
    networks:
      - factorhouse
    environment:
      CONNECT_BOOTSTRAP_SERVERS: "kafka-1:19092,kafka-2:19093,kafka-3:19094"
      CONNECT_REST_PORT: "8083"
      CONNECT_GROUP_ID: "fh-local-connect"
      CONNECT_CONFIG_STORAGE_TOPIC: "fh-local-connect-config"
      CONNECT_OFFSET_STORAGE_TOPIC: "fh-local-connect-offsets"
      CONNECT_STATUS_STORAGE_TOPIC: "fh-local-connect-status"
      CONNECT_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_KEY_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_INTERNAL_VALUE_CONVERTER: "org.apache.kafka.connect.json.JsonConverter"
      CONNECT_REST_ADVERTISED_HOST_NAME: "localhost"
      CONNECT_LOG4J_ROOT_LOGLEVEL: "INFO"
      CONNECT_PLUGIN_PATH: /usr/share/java/,/etc/kafka-connect/jars,/etc/kafka-connect/plugins
      AWS_REGION: us-east-1 # for iceberg sink
      ## OpenLineage config for custom SMT
      OPENLINEAGE_URL: http://marquez-api:5000
      OPENLINEAGE_NAMESPACE: fh-local
      KAFKA_BOOTSTRAP: "kafka-1:19092"
    volumes:
      - ./resources/deps/kafka/connector:/etc/kafka-connect/jars
      - ./resources/kpow/plugins:/etc/kafka-connect/plugins
      ## OpenLineage config for Debezium connector
      - ./resources/openlineage/kafka/openlineage.yml:/etc/kafka-connect/openlineage.yml
    healthcheck:
      test: ["CMD-SHELL", "curl -f http://localhost:8083/ || exit 1"]
      interval: 5s
      timeout: 3s
      retries: 10
      start_period: 20s

  kpow:
    image: factorhouse/kpow${KPOW_SUFFIX:-}:latest
    container_name: kpow${KPOW_SUFFIX:-}
    pull_policy: always
    restart: always
    ports:
      - "3000:3000"
      - "4000:4000"
    networks:
      - factorhouse
    depends_on:
      connect:
        condition: service_healthy
    env_file:
      - resources/kpow/config/setup.env
      - ${KPOW_LICENSE:-resources/kpow/config/license.env}
    mem_limit: 2G
    volumes:
      - ./resources/kpow/jaas:/etc/kpow/jaas
      - ./resources/kpow/rbac:/etc/kpow/rbac

  kafka-1:
    image: confluentinc/cp-kafka:8.1.1
    container_name: kafka-1
    ports:
      - "9092:9092"
      - "29092:29092"
    networks:
      - factorhouse
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_LISTENERS: CONTROLLER://kafka-1:19091,LISTENER_DOCKER_INTERNAL://kafka-1:19092,LISTENER_DOCKER_EXTERNAL://0.0.0.0:9092,LISTENER_MINIKUBE://0.0.0.0:29092
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092,LISTENER_MINIKUBE://host.minikube.internal:29092
      <<: *kafka_broker_common_env_vars

  kafka-2:
    image: confluentinc/cp-kafka:8.1.1
    container_name: kafka-2
    ports:
      - "9093:9093"
      - "29093:29093"
    networks:
      - factorhouse
    environment:
      KAFKA_BROKER_ID: 2
      KAFKA_LISTENERS: CONTROLLER://kafka-2:19091,LISTENER_DOCKER_INTERNAL://kafka-2:19093,LISTENER_DOCKER_EXTERNAL://0.0.0.0:9093,LISTENER_MINIKUBE://0.0.0.0:29093
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093,LISTENER_MINIKUBE://host.minikube.internal:29093
      <<: *kafka_broker_common_env_vars

  kafka-3:
    image: confluentinc/cp-kafka:8.1.1
    container_name: kafka-3
    ports:
      - "9094:9094"
      - "29094:29094"
    networks:
      - factorhouse
    environment:
      KAFKA_BROKER_ID: 3
      KAFKA_LISTENERS: CONTROLLER://kafka-3:19091,LISTENER_DOCKER_INTERNAL://kafka-3:19094,LISTENER_DOCKER_EXTERNAL://0.0.0.0:9094,LISTENER_MINIKUBE://0.0.0.0:29094
      KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka-3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094,LISTENER_MINIKUBE://host.minikube.internal:29094
      <<: *kafka_broker_common_env_vars

networks:
  factorhouse:
    name: factorhouse