diff --git a/docker-compose-hive.yml b/docker-compose-hive.yml index c4e6488..49f4b75 100644 --- a/docker-compose-hive.yml +++ b/docker-compose-hive.yml @@ -1,5 +1,59 @@ version: '2' services: + + zookeeper: + image: wurstmeister/zookeeper:latest + ports: + - 2181:2181 + + kafka: + image: wurstmeister/kafka:0.10.2.1 + ports: + - "9092:9092" + links: + - zookeeper + depends_on: + - zookeeper + environment: + KAFKA_BROKER_ID: 1 + KAFKA_PORT: 9092 + KAFKA_ADVERTISED_HOST_NAME: "kafka" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" + KAFKA_LISTENERS: "PLAINTEXT://:9092" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CREATE_TOPICS: "test:1:1" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + + kafka-rest: + image: nodefluent/kafka-rest + ports: + - 8082:8082 + links: + - kafka + - zookeeper + depends_on: + - kafka + - zookeeper + environment: + DEBUG: "*" + + kafka-rest-ui: + image: nodefluent/kafka-rest-ui + ports: + - 8000:8000 + links: + - kafka-rest + depends_on: + - kafka-rest + environment: + DEBUG: "*" + REACT_APP_KAFKA_REST_URL: "http://kafka-rest:8082/" + REACT_APP_TIMEOUT: "3000" + PROXY: "yes" + BASIC_AUTH_USER: "admin" + BASIC_AUTH_PASSWORD: "admin" + + namenode: image: bde2020/hadoop-namenode:1.1.0-hadoop2.8-java8 container_name: namenode diff --git a/docker-compose.yml b/docker-compose.yml index 65fcc12..1a0ec4d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,58 @@ version: '2' services: + + zookeeper: + image: wurstmeister/zookeeper:latest + ports: + - 2181:2181 + + kafka: + image: wurstmeister/kafka:0.10.2.1 + ports: + - "9092:9092" + links: + - zookeeper + depends_on: + - zookeeper + environment: + KAFKA_BROKER_ID: 1 + KAFKA_PORT: 9092 + KAFKA_ADVERTISED_HOST_NAME: "kafka" + KAFKA_ADVERTISED_LISTENERS: "PLAINTEXT://kafka:9092" + KAFKA_LISTENERS: "PLAINTEXT://:9092" + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + KAFKA_CREATE_TOPICS: "test:1:1" + KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true" + + kafka-rest: + image: nodefluent/kafka-rest + ports: + - 8082:8082 + links: + - kafka + - zookeeper + depends_on: + - kafka + - zookeeper + environment: + DEBUG: "*" + + kafka-rest-ui: + image: nodefluent/kafka-rest-ui + ports: + - 8000:8000 + links: + - kafka-rest + depends_on: + - kafka-rest + environment: + DEBUG: "*" + REACT_APP_KAFKA_REST_URL: "http://kafka-rest:8082/" + REACT_APP_TIMEOUT: "3000" + PROXY: "yes" + BASIC_AUTH_USER: "admin" + BASIC_AUTH_PASSWORD: "admin" + namenode: image: bde2020/hadoop-namenode:1.1.0-hadoop2.8-java8 container_name: namenode diff --git a/start-hadoop-spark-workbench-with-Hive.sh b/start-hadoop-spark-workbench-with-Hive.sh index 93ab6c5..8fed391 100755 --- a/start-hadoop-spark-workbench-with-Hive.sh +++ b/start-hadoop-spark-workbench-with-Hive.sh @@ -4,11 +4,12 @@ docker-compose -f docker-compose-hive.yml up -d namenode hive-metastore-postgres docker-compose -f docker-compose-hive.yml up -d datanode hive-metastore docker-compose -f docker-compose-hive.yml up -d hive-server docker-compose -f docker-compose-hive.yml up -d spark-master spark-worker spark-notebook hue +docker-compose up -d kafka kafka-rest kafka-rest-ui my_ip=`ip route get 1|awk '{print $NF;exit}'` echo "Namenode: http://${my_ip}:50070" echo "Datanode: http://${my_ip}:50075" echo "Spark-master: http://${my_ip}:8080" echo "Spark-notebook: http://${my_ip}:9001" +echo "Kafka: http://${my_ip}:8000" echo "Hue (HDFS Filebrowser): http://${my_ip}:8088/home" - diff --git a/start-hadoop-spark-workbench.sh b/start-hadoop-spark-workbench.sh index ef01035..8c00543 100755 --- a/start-hadoop-spark-workbench.sh +++ b/start-hadoop-spark-workbench.sh @@ -7,5 +7,6 @@ echo "Namenode: http://${my_ip}:50070" echo "Datanode: http://${my_ip}:50075" echo "Spark-master: http://${my_ip}:8080" echo "Spark-notebook: http://${my_ip}:9001" +echo "kafka: http://${my_ip}:8000" echo "Hue (HDFS Filebrowser): http://${my_ip}:8088/home"