EMB_datastreaming/docker-compose.yml at master · rosafilgueira/EMB_datastreaming · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
version: '2'
services:
  falcon:
    build: ./falcon
    container_name: falcon
    ports:
     - "8000:8000"
    links:
     - kafka
  zookeeper:
    image: wurstmeister/zookeeper
    hostname: zookeeper
    container_name: zookeeper
    ports:
     - "2181:2181"
  kafka:
    image: wurstmeister/kafka
    ports:
      - "9092:9092"
    hostname: kafka
    container_name: kafka
    links:
      - zookeeper
    environment:
      - KAFKA_ADVERTISED_HOST_NAME=kafka=kafka
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_ADVERTISED_PORT=9092
    volumes:
      - ./kafka_script/:/scripts
      - /var/run/docker.sock:/var/run/docker.sock

  spark-master:
    image: rosafilgueira/spark.2.2.0:latest
    container_name: spark-master
    environment:
      - MASTER=spark://spark-master:7077
      - SPARK_MASTER_HOST = spark-master
      - SPARK_DRIVER_MEMORY=2g
      - SPARK_MASTER_WEBUI_PORT=8080
    expose:
      - 7001
      - 7002
      - 7003
      - 7004
      - 7005
      - 7006
      - 7077
      - 6066
    ports:
      - "4040:4040"
      - "8080:8080"
      - "7077:7077"
      - "6066:6066"
    volumes:
      - ./docker-spark.2.2.0/conf/master:/conf
      - ./pyspark_app:/app/
      - ./pyspark_app/dependencies:/app_dependencies
      - ./pyspark_app/submit_scripts/:/submit_scripts
      - ./pyspark_app/data:/data
    command: bin/spark-class org.apache.spark.deploy.master.Master -h spark-master
    links:
      - kafka
      - elasticsearch
      - kibana
  spark-worker:
    image: rosafilgueira/spark.2.2.0:latest
    container_name: spark-worker
    environment:
      - SPARK_WORKER_CORES=2
      - SPARK_WORKER_MEMORY=1g
      - SPARK_WORKER_PORT=8881
      - SPARK_WORKER_WEBUI_PORT=8081
      - SPARK_EXECUTOR_INSTANCES=1
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_DRIVER_MEMORY=2g
    links:
      - spark-master
      - kafka
      - elasticsearch
      - kibana
    expose:
      - 7012
      - 7013
      - 7014
      - 7015
      - 7016
      - 8881
    ports:
      - "8081:8081"
    volumes:
      - ./docker-spark.2.2.0/conf/worker:/conf
      - ./pyspark_app/tmp:/test_output
      - ./pyspark_app/data:/data
      - ./pyspark_app/scripts:/scripts
    command: bin/spark-class org.apache.spark.deploy.worker.Worker spark://spark-master:7077


  elasticsearch:
    build: docker-elasticsearch/
    container_name: elasticsearch
    hostname: elasticsearch
    environment:
      - Des.network.host=0.0.0.0
      - ES_JAVA_OPTS=-Xms512m -Xmx512m
      - cluster.name = "elasticsearch"
      - discovery.zen.minimum_master_nodes= 1
      - discovery.type= single-node
    expose:
      - 9200
    ports:
      - "9200:9200"


  kibana:
    build: kibana/
    container_name: kibana
    hostname: kibana
    environment:
      - SERVER_NAME="kibana"
      - SERVER_HOST="0"
      - ELASTICSEARCH_URL=http://elasticsearch:9200
    #volumes:
    #  - ./kibana/config/kibana.yml:/usr/share/kibana/config/kibana.yml
    ports:
      - "5601:5601"
    links:
      - elasticsearch
    depends_on:
      - elasticsearch