docker-compose.yaml

services:
  zookeeper:
    image: confluentinc/cp-zookeeper:7.2.1
    hostname: zookeeper
    container_name: zookeeper
    ports:
      - 2181:2181
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
    networks:
      - iceberg_net

  broker:
    image: confluentinc/cp-server:7.2.1
    hostname: kafka
    container_name: broker
    depends_on:
      - zookeeper
    ports:
      - 9092:9092
      - 29092:29092
    healthcheck:
      test: ["CMD-SHELL", "sleep 1;"]
      interval: 30s
      timeout: 10s
      retries: 5
    environment:
      # Basic Kafka configurations
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: 'zookeeper:2181'
      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1

      # Logging settings
      KAFKA_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.apache.kafka=ERROR,kafka=ERROR,kafka.cluster=ERROR,kafka.controller=ERROR,kafka.coordinator=ERROR,kafka.log=ERROR,kafka.server=ERROR,kafka.zookeeper=ERROR,state.change.logger=ERROR
      KAFKA_LOG4J_ROOT_LOGLEVEL: ERROR

      # Metrics and telemetry configurations
      KAFKA_METRIC_REPORTERS: io.confluent.metrics.reporter.ConfluentMetricsReporter
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: broker:29092
      CONFLUENT_METRICS_ENABLE: 'false'
      CONFLUENT_TELEMETRY_ENABLED: 'false'  # Ensure telemetry is fully disabled
      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_SUPPORT_METRICS_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_TELEMETRY_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_METADATA_TOPIC_REPLICATION_FACTOR: 1

    networks:
      - iceberg_net

  control-center:
    image: confluentinc/cp-enterprise-control-center:7.2.1
    hostname: control-center
    container_name: control-center
    depends_on:
      - broker
    ports:
      - 9021:9021
    environment:
      CONTROL_CENTER_BOOTSTRAP_SERVERS: 'broker:29092'
      CONTROL_CENTER_REPLICATION_FACTOR: 1
      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
      PORT: 9021
    networks:
      - iceberg_net

  producer:
    container_name: clickstream-producer
    build:
      context: ./producer
    depends_on:
      - broker
    environment:
      KAFKA_BROKER: broker:29092
    networks:
      - iceberg_net

  ## FLINK
  jobmanager:
    container_name: jobmanager
    image: flink:1.18.1-scala_2.12-java11
    ports:
      - 18081:18081
    command: jobmanager   
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        rest.port: 18081
        state.backend: rocksdb
        state.backend.incremental: true
        AWS_REGION=us-east-1
        AWS_ACCESS_KEY_ID=admin
        AWS_SECRET_ACCESS_KEY=password
        AWS_DEFAULT_REGION=us-east-1
        S3_ENDPOINT=http://minio:9000
        S3_PATH_STYLE_ACCESS=true
        JAVA_TOOL_OPTIONS=-Daws.accessKeyId=admin -Daws.secretKey=password
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:18081"]
    networks:
      - iceberg_net

  taskmanager:
    container_name: taskmanager
    image: flink:1.18.1-scala_2.12-java11
    depends_on:
      - jobmanager
    command: taskmanager
    environment:
      - |
        FLINK_PROPERTIES=
        jobmanager.rpc.address: jobmanager
        rest.port: 18081
        taskmanager.numberOfTaskSlots: 2
        state.backend: rocksdb
        state.backend.incremental: true
        AWS_REGION=us-east-1
        AWS_ACCESS_KEY_ID=admin
        AWS_SECRET_ACCESS_KEY=password
        AWS_DEFAULT_REGION=us-east-1
        S3_ENDPOINT=http://minio:9000
        S3_PATH_STYLE_ACCESS=true
        JAVA_TOOL_OPTIONS=-Daws.accessKeyId=admin -Daws.secretKey=password

    networks:
      - iceberg_net

  sql-client:
    container_name: sql-client
    depends_on:
      jobmanager:
        condition: service_healthy
      taskmanager:
        condition: service_started
    build:
      context: ./flink/sql-client/
    environment:
      FLINK_JOBMANAGER_HOST: jobmanager
      S3_ENDPOINT: http://minio:9000
      S3_PATH_STYLE_ACCESS: true
      AWS_ACCESS_KEY_ID: admin
      AWS_SECRET_ACCESS_KEY: password
      AWS_REGION: us-east-1
      AWS_DEFAULT_REGION: us-east-1
      JAVA_TOOL_OPTIONS: -Daws.accessKeyId=admin -Daws.secretKey=password
    volumes:
      - type: bind
        source: ${PWD}/flink/sql-client/flink-conf.yaml
        target: /opt/flink/conf/flink-conf.yaml
      - type: bind
        source: ${PWD}/flink/sql-jobs/clickstream-filtering.sql
        target: /opt/flink/clickstream-filtering.sql
    command: >
      /bin/sh -c "
      /opt/flink/bin/sql-client.sh -f /opt/flink/clickstream-filtering.sql;
      tail -f /dev/null
      "
    networks:
      - iceberg_net

  minio:
    image: minio/minio
    container_name: minio
    environment:
      - MINIO_ROOT_USER=admin
      - MINIO_ROOT_PASSWORD=password
      - MINIO_DOMAIN=minio
    networks:
      iceberg_net:
        aliases:
          - warehouse.minio
    ports:
      - 9001:9001
      - 9000:9000
    command: ["server", "/data", "--console-address", ":9001"]
  mc:
    depends_on:
      - minio
    image: minio/mc
    container_name: mc
    networks:
      - iceberg_net
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - AWS_DEFAULT_REGION=us-east-1
    entrypoint: >
      /bin/sh -c "
      until (/usr/bin/mc config host add minio http://minio:9000 admin password) do echo '...waiting...' && sleep 1; done;
      /usr/bin/mc rm -r --force minio/warehouse;
      /usr/bin/mc mb minio/warehouse;
      /usr/bin/mc policy set public minio/warehouse;
      tail -f /dev/null
      "

  rest:
    image: tabulario/iceberg-rest
    container_name: iceberg-rest
    ports:
      - 8181:8181
    environment:
      - AWS_ACCESS_KEY_ID=admin
      - AWS_SECRET_ACCESS_KEY=password
      - AWS_REGION=us-east-1
      - CATALOG_WAREHOUSE=s3://warehouse/
      - CATALOG_IO__IMPL=org.apache.iceberg.aws.s3.S3FileIO
      - CATALOG_S3_ENDPOINT=http://minio:9000
    networks:
      - iceberg_net

  trino:
    image: trinodb/trino:latest
    container_name: trino
    networks:
      - iceberg_net
    environment:
      - TRINO_USER=admin
      - TRINO_PASSWORD=admin
    ports:
      - 8080:8080
    depends_on:
      - rest
      - minio
    volumes:
      - ./trino/iceberg.properties:/etc/trino/catalog/iceberg.properties

  superset:
      build:
        context: ./superset
      container_name: superset
      networks:
        - iceberg_net
      environment:
        - ADMIN_USERNAME=admin
        - ADMIN_EMAIL=admin@superset.com
        - ADMIN_PASSWORD=admin
      ports:
        - 8088:8088

networks:
  iceberg_net:
    driver: bridge

volumes:
  minio_data:
  superset-data: