diff --git a/.gitignore b/.gitignore index 8fce6030..0e6f597d 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,3 @@ data/ + +.idea/ diff --git a/Makefile b/Makefile index da7cb7d9..a9869a11 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ build: wordcount: docker build -t hadoop-wordcount ./submit docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -mkdir -p /input/ - docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.2.1/README.txt /input/ + docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -copyFromLocal -f /opt/hadoop-3.3.3/README.txt /input/ docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} hadoop-wordcount docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -cat /output/* docker run --network ${DOCKER_NETWORK} --env-file ${ENV_FILE} bde2020/hadoop-base:$(current_branch) hdfs dfs -rm -r /output diff --git a/base/Dockerfile b/base/Dockerfile index dec673e2..91f2500f 100644 --- a/base/Dockerfile +++ b/base/Dockerfile @@ -18,7 +18,7 @@ RUN curl -O https://dist.apache.org/repos/dist/release/hadoop/common/KEYS RUN gpg --import KEYS -ENV HADOOP_VERSION 3.2.1 +ENV HADOOP_VERSION 3.3.3 ENV HADOOP_URL https://www.apache.org/dist/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz RUN set -x \ diff --git a/datanode/Dockerfile b/datanode/Dockerfile index 55be14a7..237f7768 100644 --- a/datanode/Dockerfile +++ b/datanode/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 +FROM bde2020/hadoop-base:2.0.0-hadoop3.3.3-java8 MAINTAINER Ivan Ermilov diff --git a/docker-compose-v3.yml b/docker-compose-v3.yml index 84587ec9..eca1ad25 100644 --- a/docker-compose-v3.yml +++ b/docker-compose-v3.yml @@ -2,7 +2,7 @@ version: '3' services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.3.3-java8 networks: - hbase volumes: @@ -24,7 +24,7 @@ services: traefik.port: 50070 datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.3.3-java8 networks: - hbase volumes: @@ -42,7 +42,7 @@ services: traefik.port: 50075 resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.3.3-java8 networks: - hbase environment: @@ -64,7 +64,7 @@ services: disable: true nodemanager: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.3.3-java8 networks: - hbase environment: @@ -80,7 +80,7 @@ services: traefik.port: 8042 historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.3.3-java8 networks: - hbase volumes: diff --git a/docker-compose.yml b/docker-compose.yml index ed40dc62..1a5f4448 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,7 +2,7 @@ version: "3" services: namenode: - image: bde2020/hadoop-namenode:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-namenode:2.0.0-hadoop3.3.3-java8 container_name: namenode restart: always ports: @@ -16,7 +16,7 @@ services: - ./hadoop.env datanode: - image: bde2020/hadoop-datanode:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-datanode:2.0.0-hadoop3.3.3-java8 container_name: datanode restart: always volumes: @@ -27,7 +27,7 @@ services: - ./hadoop.env resourcemanager: - image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.3.3-java8 container_name: resourcemanager restart: always environment: @@ -36,7 +36,7 @@ services: - ./hadoop.env nodemanager1: - image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.3.3-java8 container_name: nodemanager restart: always environment: @@ -45,7 +45,7 @@ services: - ./hadoop.env historyserver: - image: bde2020/hadoop-historyserver:2.0.0-hadoop3.2.1-java8 + image: bde2020/hadoop-historyserver:2.0.0-hadoop3.3.3-java8 container_name: historyserver restart: always environment: diff --git a/hadoop.env b/hadoop.env index 95b3d102..230eec10 100644 --- a/hadoop.env +++ b/hadoop.env @@ -38,6 +38,6 @@ MAPRED_CONF_mapreduce_map_memory_mb=4096 MAPRED_CONF_mapreduce_reduce_memory_mb=8192 MAPRED_CONF_mapreduce_map_java_opts=-Xmx3072m MAPRED_CONF_mapreduce_reduce_java_opts=-Xmx6144m -MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ -MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ -MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.2.1/ +MAPRED_CONF_yarn_app_mapreduce_am_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.3.3/ +MAPRED_CONF_mapreduce_map_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.3.3/ +MAPRED_CONF_mapreduce_reduce_env=HADOOP_MAPRED_HOME=/opt/hadoop-3.3.3/ diff --git a/historyserver/Dockerfile b/historyserver/Dockerfile index 6ad934e2..602e7327 100644 --- a/historyserver/Dockerfile +++ b/historyserver/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 +FROM bde2020/hadoop-base:2.0.0-hadoop3.3.3-java8 MAINTAINER Ivan Ermilov diff --git a/namenode/Dockerfile b/namenode/Dockerfile index f5725ddf..dd8d597b 100644 --- a/namenode/Dockerfile +++ b/namenode/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 +FROM bde2020/hadoop-base:2.0.0-hadoop3.3.3-java8 MAINTAINER Ivan Ermilov diff --git a/nodemanager/Dockerfile b/nodemanager/Dockerfile index 966167cf..c40e5a26 100644 --- a/nodemanager/Dockerfile +++ b/nodemanager/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 +FROM bde2020/hadoop-base:2.0.0-hadoop3.3.3-java8 MAINTAINER Ivan Ermilov diff --git a/resourcemanager/Dockerfile b/resourcemanager/Dockerfile index cec9d132..9a237639 100644 --- a/resourcemanager/Dockerfile +++ b/resourcemanager/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 +FROM bde2020/hadoop-base:2.0.0-hadoop3.3.3-java8 MAINTAINER Ivan Ermilov diff --git a/submit/Dockerfile b/submit/Dockerfile index 6eba11d0..b14d098b 100644 --- a/submit/Dockerfile +++ b/submit/Dockerfile @@ -1,4 +1,4 @@ -FROM bde2020/hadoop-base:2.0.0-hadoop3.2.1-java8 +FROM bde2020/hadoop-base:2.0.0-hadoop3.3.3-java8 MAINTAINER Ivan Ermilov