(SIMP-8416) Packagecloud and rsync fixes (#132)

* Fixed:
  * rsync handling has a better check to see if rsync actually works prior to
  * using it.  The old method had the potential to try and use rsync even if it
  * no longer worked (FIPS flipped for example).
* Changed:
  * Migrated from PackageCloud to the SIMP download server for updates moving
    forward.
  * In ``install_simp_repos()``, warn instead of failing when trying to disable repos that don't exist

SIMP-8416 #comment simp-beaker-helpers update

Author: trevor-vaughan

CHANGELOG.md

@@ -1,3 +1,12 @@
+### 1.19.0 / 2020-09-30
+* Fixed:
+  * rsync handling has a better check to see if rsync actually works prior to
+    using it.  The old method had the potential to try and use rsync even if it
+    no longer worked (FIPS flipped for example).
+* Changed:
+  * Migrated from PackageCloud to the SIMP download server for updates moving
+    forward.
+
 ### 1.18.9 / 2020-08-04
 * Change windows 2012r2 VM to work around issues where the old image had
   duplicate ports trying to be opened

lib/simp/beaker_helpers.rb

@@ -30,16 +30,38 @@ def fips_enabled(sut)
              ).output.strip == '1'
   end
 
+  def rsync_functional_on?(sut)
+    # We have to check if rsync *still* works otherwise
+    return false if (@rsync_functional == false)
+
+    require 'facter'
+    unless Facter::Util::Resolution.which('rsync')
+      @rsync_functional = false
+      return @rsync_functional
+    end
+
+    require 'tempfile'
+
+    testfile = Tempfile.new('rsync_check')
+    testfile.puts('test')
+    testfile.close
+
+    begin
+      rsync_to(sut, testfile.path, sut.system_temp_path)
+    rescue Beaker::Host::CommandFailure
+      @rsync_functional = false
+      return false
+    ensure
+      testfile.unlink
+    end
+
+    return true
+  end
+
   # Figure out the best method to copy files to a host and use it
   #
   # Will create the directories leading up to the target if they don't exist
   def copy_to(sut, src, dest, opts={})
-    unless fips_enabled(sut) || @has_rsync
-      %x{which rsync 2>/dev/null}.strip
-
-      @has_rsync = !$?.nil? && $?.success?
-    end
-
     sut.mkdir_p(File.dirname(dest))
 
     if sut[:hypervisor] == 'docker'
@@ -57,7 +79,7 @@ def copy_to(sut, src, dest, opts={})
         container_id = sut.host_hash[:docker_container_id]
       end
       %x(tar #{exclude_list.join(' ')} -hcf - -C "#{File.dirname(src)}" "#{File.basename(src)}" | docker exec -i "#{container_id}" tar -C "#{dest}" -xf -)
-    elsif @has_rsync && sut.check_for_command('rsync')
+    elsif rsync_functional_on?(sut)
       # This makes rsync_to work like beaker and scp usually do
       exclude_hack = %(__-__' -L --exclude '__-__)
 
@@ -584,7 +606,7 @@ def rhel_rhsm_subscribe(sut, *opts)
   end
 
   def sosreport(sut, dest='sosreports')
-    sut.install_package('sos')
+    on(sut, 'puppet resource package sos ensure=latest')
     on(sut, 'sosreport --batch')
 
     files = on(sut, 'ls /var/tmp/sosreport* /tmp/sosreport* 2>/dev/null', :accept_all_exit_codes => true).output.lines.map(&:strip)
@@ -1170,62 +1192,70 @@ def install_puppet
     run_puppet_install_helper(install_info[:puppet_install_type], install_info[:puppet_install_version])
   end
 
-  # Configure all SIMP repos on a host and enable all but those listed in the disable list
+  # Configure all SIMP repos on a host and disable all repos in the disable Array
   #
-  # @param sut Host on which to configure SIMP repos
-  # @param disable List of SIMP repos to disable
-  # @raise if disable contains an invalid repo name.
+  # @param sut [Beaker::Host]  Host on which to configure SIMP repos
+  # @param disable [Array[String]] List of repos to disable
+  # @raise [StandardError] if disable contains an invalid repo name.
   #
   # Examples:
   #  install_simp_repos( myhost )           # install all the repos an enable them.
   #  install_simp_repos( myhost, ['simp'])  # install the repos but disable the simp repo.
   #
-  # Current set of valid SIMP repo names:
-  #  'simp'
-  #  'simp_deps'
+  # Valid repo names include any repository available on the system.
   #
-  def install_simp_repos(sut, disable = [] )
-    repos = {
-      'simp' => {
-        :baseurl   => 'https://packagecloud.io/simp-project/6_X/el/$releasever/$basearch',
-        :gpgkey    => ['https://raw.githubusercontent.com/NationalSecurityAgency/SIMP/master/GPGKEYS/RPM-GPG-KEY-SIMP',
-                    'https://download.simp-project.com/simp/GPGKEYS/RPM-GPG-KEY-SIMP-6'
-                     ],
-        :gpgcheck  => 1,
-        :sslverify => 1,
-        :sslcacert => '/etc/pki/tls/certs/ca-bundle.crt',
-        :metadata_expire => 300
-      },
-      'simp_deps' => {
-        :baseurl   => 'https://packagecloud.io/simp-project/6_X_Dependencies/el/$releasever/$basearch',
-        :gpgkey    => ['https://raw.githubusercontent.com/NationalSecurityAgency/SIMP/master/GPGKEYS/RPM-GPG-KEY-SIMP',
-                    'https://download.simp-project.com/simp/GPGKEYS/RPM-GPG-KEY-SIMP-6',
-                    'https://yum.puppet.com/RPM-GPG-KEY-puppetlabs',
-                    'https://yum.puppet.com/RPM-GPG-KEY-puppet',
-                    'https://apt.postgresql.org/pub/repos/yum/RPM-GPG-KEY-PGDG-96',
-                    'https://artifacts.elastic.co/GPG-KEY-elasticsearch',
-                    'https://grafanarel.s3.amazonaws.com/RPM-GPG-KEY-grafana',
-                    'https://dl.fedoraproject.org/pub/epel/RPM-GPG-KEY-EPEL-$releasever'
-                   ],
-        :gpgcheck  => 1,
-        :sslverify => 1,
-        :sslcacert => '/etc/pki/tls/certs/ca-bundle.crt',
-        :metadata_expire => 300
-      }
-    }
-    # Verify that the repos passed to disable are in the list of valid repos
-    disable.each { |d|
-      unless repos.has_key?(d)
-        raise("ERROR: install_simp_repo - disable contains invalid SIMP repo '#{d}'.")
+  # For backwards compatibility purposes, the following translations are
+  # automatically performed:
+  #
+  #  * 'simp'
+  #    * 'simp-community-simp'
+  #
+  #  * 'simp_deps'
+  #    * 'simp-community-epel'
+  #    * 'simp-community-postgres'
+  #    * 'simp-community-puppet'
+  #
+  def install_simp_repos(sut, disable = [])
+    # NOTE: Do *NOT* use puppet in this method since it may not be available yet
+
+    if on(sut, 'rpm -q yum-utils', :accept_all_exit_codes => true).exit_code != 0
+      on(sut, 'yum -y install yum-utils')
+    end
+
+    if on(sut, 'rpm -q simp-release-community', :accept_all_exit_codes => true).exit_code != 0
+      on(sut, 'yum -y install "https://download.simp-project.com/simp-release-community.rpm"')
+    end
+
+    to_disable = disable.dup
+
+    unless to_disable.empty?
+      if to_disable.include?('simp')
+        to_disable.delete('simp')
+        to_disable << 'simp-community-simp'
       end
-    }
-    repo_manifest = ''
-    repos.each { | repo, metadata|
-      metadata[:enabled] = disable.include?(repo) ? 0 : 1
-      repo_manifest << create_yum_resource(repo, metadata)
-    }
-    apply_manifest_on(sut, repo_manifest, :catch_failures => true)
-  end
-end
 
+      if to_disable.include?('simp_deps')
+        to_disable.delete('simp_deps')
+        to_disable << 'simp-community-epel'
+        to_disable << 'simp-community-postgres'
+        to_disable << 'simp-community-puppet'
+      end
 
+      # NOTE: This --enablerepo enables the repos for listing and is inherited
+      # from YUM. This does not actually "enable" the repos, that would require
+      # the "--enable" option (from yum-config-manager) :-D.
+      available_repos = on(sut, %{yum-config-manager --enablerepo="*"}).stdout.lines.grep(/\A\[(.+)\]\Z/){|x| $1}
+
+      invalid_repos = (to_disable - available_repos)
+
+      # Verify that the repos passed to disable are in the list of valid repos
+      unless invalid_repos.empty?
+        logger.warn(%{WARN: install_simp_repo - requested repos to disable do not exist on the target system '#{invalid_repos.join("', '")}'.})
+      end
+
+      (to_disable - invalid_repos).each do |repo|
+        on(sut, %{yum-config-manager --disable "#{repo}"})
+      end
+    end
+  end
+end

lib/simp/beaker_helpers/version.rb

@@ -1,5 +1,5 @@
 module Simp; end
 
 module Simp::BeakerHelpers
-  VERSION = '1.18.9'
+  VERSION = '1.19.0'
 end

spec/acceptance/suites/default/install_simp_deps_repo_spec.rb

@@ -2,42 +2,36 @@
 
 hosts.each do |host|
   describe '#write_hieradata_to' do
+    expect_failures = false
+    if hosts_with_role(hosts, 'el8').include?(host)
+      expect_failures = true
+    end
 
     it 'should install yum utils' do
       host.install_package('yum-utils')
     end
 
-    context 'defailt settings' do
+    context 'default settings' do
       before(:all) { install_simp_repos(host) }
 
-      it 'creates the repo' do
-        on host, 'test -f /etc/yum.repos.d/simp.repo'
-        on host, 'test -f /etc/yum.repos.d/simp_deps.repo'
-      end
-
       it 'enables the correct repos' do
-        simp6info = on(host, '/usr/bin/yum repolist -v simp | grep ^Repo-status').stdout.strip
-        expect(simp6info).to match(/.*Repo-status.*enabled.*/)
-        simp6depsinfo = on(host, 'yum repolist -v simp_deps| grep ^Repo-status').stdout.strip
-        expect(simp6depsinfo).to match(/.*Repo-status.*enabled.*/)
+        skip "#{host} is not supported yet" if expect_failures
+        on(host, 'yum -y list simp')
+        on(host, 'yum -y list postgresql96')
       end
     end
 
     context 'when passed a disabled list ' do
-      before(:all) { install_simp_repos(host, ['simp'] ) }
+      before(:all) { install_simp_repos(host, ['simp-community-simp'] ) }
 
-      it 'creates the repo' do
-        on host, 'test -f /etc/yum.repos.d/simp.repo'
-        on host, 'test -f /etc/yum.repos.d/simp_deps.repo'
+      it 'enables the correct repos' do
+        skip "#{host} is not supported yet" if expect_failures
+        on(host, 'yum -y list postgresql96')
       end
 
-      it 'enables the correct repos' do
-        simp6info = on(host, 'yum repolist -v simp | grep ^Repo-status').stdout.strip
-        expect(simp6info).to match(/.*Repo-status.*disabled.*/)
-        simp6depsinfo = on(host, 'yum repolist -v simp_deps| grep ^Repo-status').stdout.strip
-        expect(simp6depsinfo).to match(/.*Repo-status.*enabled.*/)
+      it 'disables the correct repos' do
+        on(host, 'yum -y list simp', :acceptable_exit_codes => [1])
       end
     end
-
   end
 end