From 4a303338285b62483ec2ee088871d04207bdc423 Mon Sep 17 00:00:00 2001 From: Owen O'Malley Date: Wed, 13 May 2020 16:01:54 -0700 Subject: [PATCH] Update to HDP 2.6.5. Also upgrades out of necessity to openjdk8. Removes lzo since the HDP GPL repository seems to be unavailable. --- Vagrantfile | 4 ++-- files/repos/centos6.hdp.repo.2.6.5 | 24 +++++++++++++++++++ .../ambari_agent/templates/ambari-agent.erb | 2 +- modules/hdfs_client/manifests/init.pp | 13 ---------- modules/hdfs_client/templates/core-site.erb | 7 +----- modules/jdk/manifests/init.pp | 4 ++-- modules/oozie_server/manifests/init.pp | 5 ---- modules/tez_client/templates/tez-site.erb | 2 +- modules/yarn_client/templates/mapred-site.erb | 2 +- modules/yarn_client/templates/yarn-site.erb | 2 +- 10 files changed, 33 insertions(+), 32 deletions(-) create mode 100644 files/repos/centos6.hdp.repo.2.6.5 diff --git a/Vagrantfile b/Vagrantfile index 4a3d8cc..5293f06 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -68,9 +68,9 @@ profile = loadProfile() # Set defaults. default_os = "centos6" -default_hdp_short_version = "2.6.1" +default_hdp_short_version = "2.6.5" default_ambari_version = "2.4.2" -default_java_version = "java-1.7.0-openjdk" +default_java_version = "java-1.8.0-openjdk" profile[:hdp_short_version] ||= default_hdp_short_version profile[:ambari_version] ||= default_ambari_version diff --git a/files/repos/centos6.hdp.repo.2.6.5 b/files/repos/centos6.hdp.repo.2.6.5 new file mode 100644 index 0000000..da1cc89 --- /dev/null +++ b/files/repos/centos6.hdp.repo.2.6.5 @@ -0,0 +1,24 @@ +#VERSION_NUMBER=2.6.5.0-292 +[HDP-2.6.5.0] +name=HDP Version - HDP-2.6.5.0 +baseurl=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.5.0 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.5.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + +[HDP-UTILS-1.1.0.22] +name=HDP-UTILS Version - HDP-UTILS-1.1.0.22 +baseurl=http://public-repo-1.hortonworks.com/HDP-UTILS-1.1.0.22/repos/centos6 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.5.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 + +[HDP-UTILS-GPL-1.1.0.22] +name=HDP-UTILS Version - HDP-UTILS-GPL-1.1.0.22 +baseurl=http://public-repo-1.hortonworks.com/HDP-UTILS-GPL-1.1.0.22/repos/centos6 +gpgcheck=1 +gpgkey=http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.6.5.0/RPM-GPG-KEY/RPM-GPG-KEY-Jenkins +enabled=1 +priority=1 diff --git a/modules/ambari_agent/templates/ambari-agent.erb b/modules/ambari_agent/templates/ambari-agent.erb index 6afd459..9ba7852 100644 --- a/modules/ambari_agent/templates/ambari-agent.erb +++ b/modules/ambari_agent/templates/ambari-agent.erb @@ -63,7 +63,7 @@ dirs=/etc/hadoop,/etc/hadoop/conf,/etc/hbase,/etc/hcatalog,/etc/hive,/etc/oozie, /var/log/hadoop,/var/log/zookeeper,/var/log/hbase,/var/run/templeton,/var/log/hive, /var/log/nagios rpms=nagios,ganglia, - hadoop,hadoop-lzo,hbase,oozie,sqoop,pig,zookeeper,hive,libconfuse,ambari-log4j + hadoop,hbase,oozie,sqoop,pig,zookeeper,hive,libconfuse,ambari-log4j ; 0 - unlimited log_lines_count=300 idle_interval_min=1 diff --git a/modules/hdfs_client/manifests/init.pp b/modules/hdfs_client/manifests/init.pp index 3a46c90..b6fd9d6 100644 --- a/modules/hdfs_client/manifests/init.pp +++ b/modules/hdfs_client/manifests/init.pp @@ -44,15 +44,6 @@ require => Package["hadoop${package_version}"], } - package { "hadooplzo${package_version}": - ensure => installed, - require => Package["hadoop${package_version}"], - } - -> - package { "hadooplzo${package_version}-native": - ensure => installed, - } - package { 'openssl': ensure => installed, } @@ -61,10 +52,6 @@ ensure => installed, } - package { 'lzo': - ensure => installed, - } - file { '/etc/hadoop': ensure => 'directory', } diff --git a/modules/hdfs_client/templates/core-site.erb b/modules/hdfs_client/templates/core-site.erb index 8598d9e..eb35639 100755 --- a/modules/hdfs_client/templates/core-site.erb +++ b/modules/hdfs_client/templates/core-site.erb @@ -56,12 +56,7 @@ io.compression.codecs - org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,com.hadoop.compression.lzo.LzoCodec,com.hadoop.compression.lzo.LzopCodec,org.apache.hadoop.io.compress.SnappyCodec - - - - io.compression.codec.lzo.class - com.hadoop.compression.lzo.LzoCodec + org.apache.hadoop.io.compress.GzipCodec,org.apache.hadoop.io.compress.DefaultCodec,org.apache.hadoop.io.compress.SnappyCodec diff --git a/modules/jdk/manifests/init.pp b/modules/jdk/manifests/init.pp index b6df7c5..3149ef3 100644 --- a/modules/jdk/manifests/init.pp +++ b/modules/jdk/manifests/init.pp @@ -16,11 +16,11 @@ class jdk { $home = "/usr/lib/jvm/java" - package { "java-1.7.0-openjdk": + package { "java-1.8.0-openjdk": ensure => installed, } - package { "java-1.7.0-openjdk-devel": + package { "java-1.8.0-openjdk-devel": ensure => installed, } diff --git a/modules/oozie_server/manifests/init.pp b/modules/oozie_server/manifests/init.pp index 9544fe4..57f9f45 100644 --- a/modules/oozie_server/manifests/init.pp +++ b/modules/oozie_server/manifests/init.pp @@ -94,11 +94,6 @@ target => "/usr/share/HDP-oozie/ext-2.2.zip", } -> - file { "/usr/hdp/${hdp_version}/oozie/libext/hadoop-lzo.jar": - ensure => link, - target => "/usr/hdp/${hdp_version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp_version}.jar", - } - -> file { "/usr/hdp/${hdp_version}/oozie/libext/mysql-connector-java.jar": ensure => link, target => "/usr/share/java/mysql-connector-java.jar", diff --git a/modules/tez_client/templates/tez-site.erb b/modules/tez_client/templates/tez-site.erb index f78c898..f350ed4 100644 --- a/modules/tez_client/templates/tez-site.erb +++ b/modules/tez_client/templates/tez-site.erb @@ -28,7 +28,7 @@ tez.cluster.additional.classpath.prefix - /usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar:/etc/hadoop/conf + /etc/hadoop/conf diff --git a/modules/yarn_client/templates/mapred-site.erb b/modules/yarn_client/templates/mapred-site.erb index 5f96012..181fe40 100755 --- a/modules/yarn_client/templates/mapred-site.erb +++ b/modules/yarn_client/templates/mapred-site.erb @@ -31,7 +31,7 @@ mapreduce.application.classpath - mapreduce/hadoop/share/hadoop/mapreduce/*:mapreduce/hadoop/share/hadoop/mapreduce/lib/*:mapreduce/hadoop/share/hadoop/common/*:mapreduce/hadoop/share/hadoop/common/lib/*:mapreduce/hadoop/share/hadoop/yarn/*:mapreduce/hadoop/share/hadoop/yarn/lib/*:mapreduce/hadoop/share/hadoop/hdfs/*:mapreduce/hadoop/share/hadoop/hdfs/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar + mapreduce/hadoop/share/hadoop/mapreduce/*:mapreduce/hadoop/share/hadoop/mapreduce/lib/*:mapreduce/hadoop/share/hadoop/common/*:mapreduce/hadoop/share/hadoop/common/lib/*:mapreduce/hadoop/share/hadoop/yarn/*:mapreduce/hadoop/share/hadoop/yarn/lib/*:mapreduce/hadoop/share/hadoop/hdfs/*:mapreduce/hadoop/share/hadoop/hdfs/lib/* diff --git a/modules/yarn_client/templates/yarn-site.erb b/modules/yarn_client/templates/yarn-site.erb index 37814e2..4a9be8f 100644 --- a/modules/yarn_client/templates/yarn-site.erb +++ b/modules/yarn_client/templates/yarn-site.erb @@ -226,7 +226,7 @@ yarn.application.classpath - /usr/hdp/${hdp.version}/hadoop/conf,/usr/hdp/${hdp.version}/hadoop/*,/usr/hdp/${hdp.version}/lib/*,/usr/hdp/${hdp.version}/hadoop-hdfs/*,/usr/hdp/${hdp.version}/hadoop-hdfs/lib/*,/usr/hdp/${hdp.version}/hadoop-yarn/*,/usr/hdp/${hdp.version}/hadoop-yarn/lib/*,/usr/hdp/${hdp.version}/hadoop-mapreduce/*,/usr/hdp/${hdp.version}/hadoop-mapreduce/lib/*:/usr/hdp/${hdp.version}/hadoop/lib/hadoop-lzo-0.6.0.${hdp.version}.jar + /usr/hdp/${hdp.version}/hadoop/conf,/usr/hdp/${hdp.version}/hadoop/*,/usr/hdp/${hdp.version}/lib/*,/usr/hdp/${hdp.version}/hadoop-hdfs/*,/usr/hdp/${hdp.version}/hadoop-hdfs/lib/*,/usr/hdp/${hdp.version}/hadoop-yarn/*,/usr/hdp/${hdp.version}/hadoop-yarn/lib/*,/usr/hdp/${hdp.version}/hadoop-mapreduce/*,/usr/hdp/${hdp.version}/hadoop-mapreduce/lib/*