diff --git a/Makefile b/Makefile index 62087dd..4d3370f 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ DOCKER_NETWORK = hbase ENV_FILE = hadoop.env current_branch := $(shell git rev-parse --abbrev-ref HEAD) -hadoop_branch := 2.0.0-hadoop2.7.4-java8 +hadoop_branch := 2.0.0-hadoop3.2.1-java8 build: docker build -t bde2020/hbase-base:$(current_branch) ./base docker build -t bde2020/hbase-master:$(current_branch) ./hmaster @@ -10,7 +10,7 @@ build: wordcount: docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(hadoop_branch) hdfs dfs -mkdir -p /input/ - docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(hadoop_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-2.7.4/README.txt /input/ + docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(hadoop_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/ docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(hadoop_branch) hdfs dfs -cat /output/* docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(hadoop_branch) hdfs dfs -rm -r /output diff --git a/base/Dockerfile b/base/Dockerfile index e431c2b..c36a3dc 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -3,7 +3,7 @@ MAINTAINER Ivan Ermilov RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends net-tools curl netcat -ENV HBASE_VERSION 1.2.6 +ENV HBASE_VERSION 2.2.5 ENV HBASE_URL http://www.apache.org/dist/hbase/$HBASE_VERSION/hbase-$HBASE_VERSION-bin.tar.gz RUN set -x \ && curl -fSL "$HBASE_URL" -o /tmp/hbase.tar.gz \ diff --git a/distributed/docker-compose-hadoop.yml b/distributed/docker-compose-hadoop.yml index a8d6b52..93be47d 100644 --- a/distributed/docker-compose-hadoop.yml +++ b/distributed/docker-compose-hadoop.yml @@ -2,7 +2,7 @@ version: '3' services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 networks: - hbase volumes: @@ -21,10 +21,10 @@ services: - node.hostname == akswnc4.aksw.uni-leipzig.de labels: traefik.docker.network: hbase - traefik.port: 50070 + traefik.port: 9870 datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 networks: - hbase volumes: @@ -32,21 +32,21 @@ services: env_file: - ./hadoop.env environment: - SERVICE_PRECONDITION: "namenode:50070" + SERVICE_PRECONDITION: "namenode:9870" deploy: mode: global restart_policy: condition: on-failure labels: traefik.docker.network: hbase - traefik.port: 50075 + traefik.port: 9864 resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 networks: - hbase environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075" + SERVICE_PRECONDITION: "namenode:9870 datanode:9864" env_file: - ./hadoop.env deploy: @@ -64,11 +64,11 @@ services: disable: true nodemanager: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8 networks: - hbase environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" + SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" env_file: - ./hadoop.env deploy: @@ -80,13 +80,13 @@ services: traefik.port: 8042 historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8 networks: - hbase volumes: - hadoop_historyserver:/hadoop/yarn/timeline environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" + SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" env_file: - ./hadoop.env deploy: diff --git a/distributed/docker-compose-hbase.yml b/distributed/docker-compose-hbase.yml index 5729192..aaa68be 100644 --- a/distributed/docker-compose-hbase.yml +++ b/distributed/docker-compose-hbase.yml @@ -2,7 +2,7 @@ version: '3.2' services: hbase-master: - image: bde2020/hbase-master:1.0.0-hbase1.2.6 + image: bde2020/hbase-master:2.0.0-hbase2.2.5 networks: - hbase deploy: @@ -21,7 +21,7 @@ services: - ./hbase.env hbase-regionserver-1: - image: bde2020/hbase-regionserver:1.0.0-hbase1.2.6 + image: bde2020/hbase-regionserver:2.0.0-hbase2.2.5 networks: - hbase deploy: @@ -39,7 +39,7 @@ services: HBASE_CONF_hbase_regionserver_hostname: hbase-regionserver-1 hbase-regionserver-2: - image: bde2020/hbase-regionserver:1.0.0-hbase1.2.6 + image: bde2020/hbase-regionserver:2.0.0-hbase2.2.5 networks: - hbase deploy: @@ -57,7 +57,7 @@ services: HBASE_CONF_hbase_regionserver_hostname: hbase-regionserver-2 hbase-regionserver-3: - image: bde2020/hbase-regionserver:1.0.0-hbase1.2.6 + image: bde2020/hbase-regionserver:2.0.0-hbase2.2.5 networks: - hbase deploy: diff --git a/docker-compose-distributed-local.yml b/docker-compose-distributed-local.yml index 106f8ca..112fb74 100644 --- a/docker-compose-distributed-local.yml +++ b/docker-compose-distributed-local.yml @@ -2,7 +2,7 @@ version: "3" services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 container_name: namenode volumes: - hadoop_namenode:/hadoop/dfs/name @@ -11,47 +11,47 @@ services: env_file: - ./hadoop.env ports: - - 50070:50070 + - 9870:9870 datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 container_name: datanode volumes: - hadoop_datanode:/hadoop/dfs/data environment: - SERVICE_PRECONDITION: "namenode:50070" + SERVICE_PRECONDITION: "namenode:9870" env_file: - ./hadoop.env ports: - - 50075:50075 + - 9864:9864 resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 container_name: resourcemanager environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075" + SERVICE_PRECONDITION: "namenode:9870 datanode:9864" env_file: - ./hadoop.env ports: - 8088:8088 nodemanager1: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8 container_name: nodemanager environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" + SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" env_file: - ./hadoop.env ports: - 8042:8042 historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8 container_name: historyserver volumes: - hadoop_historyserver:/hadoop/yarn/timeline environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" + SERVICE_PRECONDITION: "namenode:9870 datanode:9864 resourcemanager:8088" env_file: - ./hadoop.env ports: @@ -67,25 +67,25 @@ services: - 2181:2181 hbase-master: - image: bde2020/hbase-master:1.0.0-hbase1.2.6 + image: bde2020/hbase-master:2.0.0-hbase2.2.5 container_name: hbase-master hostname: hbase-master env_file: - ./hbase-distributed-local.env environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075 zoo:2181" + SERVICE_PRECONDITION: "namenode:9870 datanode:9864 zoo:2181" ports: - 16010:16010 hbase-region: - image: bde2020/hbase-regionserver:1.0.0-hbase1.2.6 + image: bde2020/hbase-regionserver:2.0.0-hbase2.2.5 container_name: hbase-regionserver hostname: hbase-regionserver env_file: - ./hbase-distributed-local.env environment: HBASE_CONF_hbase_regionserver_hostname: hbase-region - SERVICE_PRECONDITION: "namenode:50070 datanode:50075 zoo:2181 hbase-master:16010" + SERVICE_PRECONDITION: "namenode:9870 datanode:9864 zoo:2181 hbase-master:16010" ports: - 16030:16030 diff --git a/docker-compose-standalone.yml b/docker-compose-standalone.yml index 9f33bbe..9d009e1 100644 --- a/docker-compose-standalone.yml +++ b/docker-compose-standalone.yml @@ -2,7 +2,7 @@ version: "3" services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 container_name: namenode volumes: - hadoop_namenode:/hadoop/dfs/name @@ -12,43 +12,43 @@ services: - ./hadoop.env datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 container_name: datanode volumes: - hadoop_datanode:/hadoop/dfs/data environment: - SERVICE_PRECONDITION: "namenode:50070" + SERVICE_PRECONDITION: "namenode:9870" env_file: - ./hadoop.env resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 container_name: resourcemanager environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075" + SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864" env_file: - ./hadoop.env nodemanager1: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8 container_name: nodemanager environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" + SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088" env_file: - ./hadoop.env historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop2.7.4-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8 container_name: historyserver volumes: - hadoop_historyserver:/hadoop/yarn/timeline environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075 resourcemanager:8088" + SERVICE_PRECONDITION: "namenode:9000 namenode:9870 datanode:9864 resourcemanager:8088" env_file: - ./hadoop.env hbase: - image: bde2020/hbase-standalone:1.0.0-hbase1.2.6 + image: bde2020/hbase-standalone:2.0.0-hbase2.2.5 container_name: hbase volumes: - hbase_data:/hbase-data @@ -62,7 +62,7 @@ services: - 3888:3888 - 2181:2181 environment: - SERVICE_PRECONDITION: "namenode:50070 datanode:50075" + SERVICE_PRECONDITION: "namenode:9000 datanode:9864" env_file: - ./hbase-standalone.env diff --git a/hmaster/Dockerfile b/hmaster/Dockerfile index ab9090a..1bf398d 100644 --- a/hmaster/Dockerfile +++ b/hmaster/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hbase-base:1.0.0-hbase1.2.6 +FROM bde2020/hbase-base:2.0.0-hbase2.2.5 COPY run.sh /run.sh RUN chmod +x /run.sh diff --git a/hregionserver/Dockerfile b/hregionserver/Dockerfile index 0c4ea88..10eac8b 100644 --- a/hregionserver/Dockerfile +++ b/hregionserver/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hbase-base:1.0.0-hbase1.2.6 +FROM bde2020/hbase-base:2.0.0-hbase2.2.5 COPY run.sh /run.sh RUN chmod +x /run.sh diff --git a/standalone/Dockerfile b/standalone/Dockerfile index eceff0b..a9de9d8 100644 --- a/standalone/Dockerfile +++ b/standalone/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hbase-base:1.0.0-hbase1.2.6 +FROM bde2020/hbase-base:2.0.0-hbase2.2.5 COPY run.sh /run.sh RUN chmod +x /run.sh