-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathdocker-compose.yml
121 lines (112 loc) · 3.37 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
services:
zoo1:
image: confluentinc/cp-zookeeper:latest
hostname: zoo1
container_name: zoo1
ports:
- "2181:2181"
environment:
ZOOKEEPER_CLIENT_PORT: 2181
kafka1:
image: confluentinc/cp-kafka:latest
hostname: kafka1
container_name: kafka1
ports:
- "9092:9092"
environment:
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
KAFKA_BROKER_ID: 1
depends_on:
- zoo1
kafka2:
image: confluentinc/cp-kafka:latest
hostname: kafka2
container_name: kafka2
ports:
- "9093:9093"
environment:
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka2:19093,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9093
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
KAFKA_BROKER_ID: 2
depends_on:
- zoo1
kafka3:
image: confluentinc/cp-kafka:latest
hostname: kafka3
container_name: kafka3
ports:
- "9094:9094"
environment:
KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka3:19094,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9094
KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT
KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL
KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181"
KAFKA_BROKER_ID: 3
depends_on:
- zoo1
kafka-ui:
image: provectuslabs/kafka-ui:latest
container_name: kafka-ui
ports:
- "8081:8080"
depends_on:
- kafka1
- kafka2
- kafka3
- zoo1
environment:
- KAFKA_CLUSTERS_0_NAME=local
- KAFKA_CLUSTERS_0_BOOTSTRAPSERVERS=kafka1:19092,kafka2:19093,kafka3:19094
- KAFKA_CLUSTERS_0_ZOOKEEPER=zoo1:2181
mongo:
image: mongo:latest
container_name: mongodb
restart: always
ports:
- "27017:27017"
environment:
MONGO_INITDB_DATABASE: usa2024
volumes:
- mongo-data:/data/db
- ./mongo/init-mongo-db-with-collections.js:/docker-entrypoint-initdb.d/init-mongo-db-with-collections.js:ro
dashboard:
build: ./dashboard
container_name: dashboard
ports:
- "1323:1323"
depends_on:
- mongo
environment:
- MONGO_URI=mongodb://mongodb:27017
consumer:
build: ./consumer
container_name: consumer
depends_on:
- kafka1
- kafka2
- kafka3
- mongo
environment:
- SPARK_DRIVER_MEMORY=4g # Increase this as needed
- SPARK_EXECUTOR_MEMORY=4g # Increase this as needed
- KAFKA_BOOTSTRAP_SERVER=kafka1:19092,kafka2:19093,kafka3:19094
- MONGO_URI=mongodb://mongodb:27017
producer:
build: ./kafka
container_name: producer
depends_on:
- kafka1
- kafka2
- kafka3
- consumer
environment:
- KAFKA_BOOTSTRAP_SERVER=kafka1:19092,kafka2:19093,kafka3:19094
volumes:
- ./twitter-scraper:/code/twitter-scraper
volumes:
mongo-data: