spark-kafka-stream/docker-compose.yml at main · ysano/spark-kafka-stream · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
version: '3.8'

services:
  spark:
    image: docker.io/bitnami/spark:2.4.6
    environment:
      - SPARK_MODE=master
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    ports:
      - '8080:8080'
    expose:
      - '7077'
    # command: /opt/bitnami/spark/app/startup.sh
    volumes:
      - ..:/opt/spark-book
      # fix image bug
      - ./jars_dir:/opt/bitnami/spark/ivy:z
      # dependency jars dir
      - ./jars_dir:/opt/bitnami/spark/jars/ivy:z
      - ./spark_data/node-1-conf:/opt/bitnami/spark/conf
      - ./spark_data/app:/opt/bitnami/spark/app
    depends_on:
      - cassandra-node1
  spark-worker-1:
    image: docker.io/bitnami/spark:2.4.6
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=1G
      - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    volumes:
      - ./jars_dir:/opt/bitnami/spark/ivy:z
      - ./jars_dir:/opt/bitnami/spark/jars/ivy:z
      - ./spark_data/node-2-conf:/opt/bitnami/spark/conf
  spark-worker-2:
    image: docker.io/bitnami/spark:2.4.6
    environment:
      - SPARK_MODE=worker
      - SPARK_MASTER_URL=spark://spark:7077
      - SPARK_WORKER_MEMORY=1G
      - SPARK_WORKER_CORES=1
      - SPARK_RPC_AUTHENTICATION_ENABLED=no
      - SPARK_RPC_ENCRYPTION_ENABLED=no
      - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
      - SPARK_SSL_ENABLED=no
    volumes:
      - ./jars_dir:/opt/bitnami/spark/ivy:z
      - ./jars_dir:/opt/bitnami/spark/jars/ivy:z
      - ./spark_data/node-3-conf:/opt/bitnami/spark/conf
  cassandra-node1:
    image: docker.io/bitnami/cassandra:3
    expose:
      - '7000'
      - '9042'
    environment:
      - CASSANDRA_CLUSTER_NAME=cassandra-cluster
      - CASSANDRA_SEEDS=cassandra-node1,cassandra-node2,cassandra-node3
      - CASSANDRA_PASSWORD_SEEDER=yes
      - CASSANDRA_PASSWORD=cassandra
    volumes:
      - ./schema:/docker-entrypoint-initdb.d
  cassandra-node2:
    image: docker.io/bitnami/cassandra:3
    expose:
      - '7000'
      - '9042'
    environment:
      - CASSANDRA_CLUSTER_NAME=cassandra-cluster
      - CASSANDRA_SEEDS=cassandra-node1,cassandra-node2,cassandra-node3
      - CASSANDRA_PASSWORD=cassandra
  cassandra-node3:
    image: docker.io/bitnami/cassandra:3
    expose:
      - '7000'
      - '9042'
    environment:
      - CASSANDRA_CLUSTER_NAME=cassandra-cluster
      - CASSANDRA_SEEDS=cassandra-node1,cassandra-node2,cassandra-node3
      - CASSANDRA_PASSWORD=cassandra
  fluentd_forward:
    image: docker.io/bitnami/fluentd:1
    ports:
      - '24224:24224'
      - '5140:5140'
  fluentd_http:
    image: docker.io/bitnami/fluentd:1
    ports:
      - '9999:9999'
    volumes:
      - ./fluentd_data/conf:/opt/bitnami/fluentd/conf
      - ./fluentd_data/pos:/opt/bitnami/fluentd/logs/fluentd
      - ./sensor_data:/opt/bitnami/fluentd/logs/sensor_data
  zookeeper:
    image: bitnami/zookeeper:latest
    ports:
      - '2181:2181'
    environment:
      - ALLOW_ANONYMOUS_LOGIN=yes
  kafka:
    image: bitnami/kafka:latest
    ports:
      - '9092:9092'
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_CFG_LISTENERS=PLAINTEXT://kafka:9092
      - KAFKA_CFG_ADVERTISED_LISTENERS=PLAINTEXT://kafka:9092
      - KAFKA_CFG_ZOOKEEPER_CONNECT=zookeeper:2181
      - ALLOW_PLAINTEXT_LISTENER=yes
    depends_on:
      - zookeeper
  pyspark-notebook:
    image: jupyter/pyspark-notebook:spark-2
    ports:
      - '8888:8888'
      - '4040:4040'
      - '4041:4041'
      - '4042:4042'
    environment:
      - JUPYTER_ENABLE_LAB=yes
      - GRANT_SUDO=yes
    volumes:
      - ..:/opt/spark-book
      - ./notebook_data/work:/home/jovyan/work
      - ./notebook_data/startup:/home/jovyan/.ipython/profile_default/startup
      - ./notebook_data/conf:/usr/local/spark/conf