Merge branch 'knobcone' into 'master'

Merge knobcone release branch to main branch.

Closes VIYAARK-237

See merge request sassoftware/viya-ark!358

Author: kevinlinglesas

CHANGELOG.md

@@ -1,6 +1,29 @@
 # Changelog for SAS Viya ARK
 
 <!-- LATEST RELEASE START -->
+## Viya35-ark-1.12 - February 16, 2021 
+- **Summary**:
+  RHEL8 Support and bug fixes.
+- Issues addressed:
+  - SAS Multi-Machine Service Utilties
+    - VIYAARK-226 - MMSU: Validate RHEL8 
+  - SAS Viya Upgrade Tasks Playbooks
+    - VIYAARK-227 - Validate RHEL8
+    - VIYAARK-237 - Merge playbook using check_invalid_arguments which was removed in Ansible 2.10
+    - VIYAARK-240 - Merge Playbook ConfigParser Python2 Python3 Compatibility ()
+  - SAS Viya Pre-Installation Playbook
+    - VIYAARK-218 - Fails on clientaliveinterval control even when you run the playbook with the exclude option. (GitHub Issue #69)
+    - VIYAARK-224 - Validate RHEL8
+    - VIYAARK-229 - Get the permissions of the directories TASK fails in checkmode
+    - VIYAARK-232 - Check for RHEL8 compatibility packages
+    - VIYAARK-238 - Support RHEL8 OS checks
+  - SAS Viya Deployment Report
+    - VIYAARK-225 - Validate RHEL8
+    - VIYAARK-239 - UnicodeEncodeError: 'ascii' codec can't encode character correctly. (GitHub Issue #75)
+- Ansible Support: Ansible 2.8 - Ansible 2.10
+
+<!-- LATEST RELEASE END -->
+
 ## Viya35-ark-1.11 - December 18, 2020 
 - **Summary**:
   Bug fixes and a couple of enhancments.
@@ -14,7 +37,6 @@
     - VIYAARK-222 - Archive Obsolete Log Folders Playbook Fails on Power Linux
 - Ansible Support: Ansible 2.8 - Ansible 2.10
 
-<!-- LATEST RELEASE END -->
 
 ## Viya35-ark-1.10 - October 20, 2020 
 - **Summary**:

playbooks/deployment-report/library/get_sas_host_details.py

@@ -10,6 +10,7 @@
 # SPDX-License-Identifier: Apache-2.0
 #
 import os
+import sys
 import datetime
 import glob
 import subprocess
@@ -44,6 +45,12 @@
             This option will greatly increase the size of the data being returned.
         default: false
         required: false
+    locale_encoding:
+        description: >
+            Encoding to use for text returned to the module that may be localized for the locale of the
+            deployed software. The value must be an encoding supported by Python.
+        default: None
+        required: false
 '''
 
 EXAMPLES = '''
@@ -57,6 +64,12 @@
   get_sas_host_details:
     hostvars: "{{ hostvars[inventory_hostname] }}"
     include_package_files: true
+
+# Get SAS deployment information with custom locale encoding
+- name: Inspect SAS deployment
+  get_sas_host_details:
+    hostvars: "{{ hostvars[inventory_hostname] }}"
+    locale_encoding: "macroman"
 '''
 
 RETURN = '''
@@ -751,6 +764,7 @@ class _ModuleParamKeys(object):
     """
     HOSTVARS = 'hostvars'
     INCL_PKG_FILES = 'include_package_files'
+    LOCALE_ENCODING = 'locale_encoding'
 
 
 # =====
@@ -895,13 +909,15 @@ def main():
     # supports check mode
     module = AnsibleModule(
         argument_spec={_ModuleParamKeys.HOSTVARS: dict(type='raw', required=True),
-                       _ModuleParamKeys.INCL_PKG_FILES: dict(type=bool, default=False, required=False)},
+                       _ModuleParamKeys.INCL_PKG_FILES: dict(type=bool, default=False, required=False),
+                       _ModuleParamKeys.LOCALE_ENCODING: dict(type='str', default=None, required=False)},
         supports_check_mode=True
     )
 
     # get module parameters
     hostvars = module.params[_ModuleParamKeys.HOSTVARS]
     include_package_files = module.params[_ModuleParamKeys.INCL_PKG_FILES]
+    locale_encoding = module.params[_ModuleParamKeys.LOCALE_ENCODING]
 
     # Starting in Ansible 2.8.1, there is the potential for hostvars
     # to be passed as a byte string, if the dict is too large
@@ -1435,26 +1451,27 @@ def _get_process_memory_info(pid, module):
 # =====
 # _get_sas_package_info(AnsibleModule, bool)
 # =====
-def _get_sas_package_info(module, package_manager, include_installed_files=False):
+def _get_sas_package_info(module, package_manager, include_installed_files=False, locale_encoding=False):
     """
     Retrieves package information for all installed SAS packages and returns the values as a dict.
 
     :param AnsibleModule module: The AnsibleModule object representing the current module.
     :param bool include_installed_files: Toggles whether information about the package's installed files should be
                                        included.
+    :param locale_encoding: A custom locale encoding that should be used to decode potentially localized text.
     :return: A dict of package names mapped to a dict containing package attribute names and their values.
     :rtype dict:
     """
 
     # -- package info -- #
 
     # get installed package info via rpm
-    results = _get_installed_package_info(module, include_installed_files)
+    results = _get_installed_package_info(module, include_installed_files, locale_encoding)
 
     if package_manager == _PackageManagers.ZYPPER:
-        update_results = _get_sas_package_update_info_zypper(module)
+        update_results = _get_sas_package_update_info_zypper(module, locale_encoding)
     else:
-        update_results = _get_sas_package_update_info_yum(module)
+        update_results = _get_sas_package_update_info_yum(module, locale_encoding)
 
     for package_name, update_info in update_results.items():
         results[package_name][_HostDetailsKeys.SASPackageKeys.UPDATE_STATUS] = update_info
@@ -1466,13 +1483,14 @@ def _get_sas_package_info(module, package_manager, include_installed_files=False
 # =====
 # _get_installed_package_info(AnsibleModule)
 # =====
-def _get_installed_package_info(module, include_installed_files):
+def _get_installed_package_info(module, include_installed_files, locale_encoding):
     """
     Retrieves package information via rpm for all installed SAS Packages and returns the values as a dict.
 
     :param module: The AnsibleModule object representing the current module.
     :param bool include_installed_files: Toggles whether information about the package's installed files should be
                                        included.
+    :param locale_encoding: A custom locale encoding that should be used to decode potentially localized text.
     :return: A dict of package names mapped to a dict of package attributes and their values.
     :rtype dict:
     """
@@ -1509,7 +1527,8 @@ def _get_installed_package_info(module, include_installed_files):
         # get the package name (first line in each block of output)
         try:
             package_name_line = package_info_lines.pop(0)
-            package_name_attr = str(package_name_line).split(delim)
+            package_name_line = _decode_str(package_name_line, locale_encoding)
+            package_name_attr = package_name_line.split(delim)
             if len(package_name_attr) == 2 and package_name_attr[0] == \
                     _HostDetailsKeys.SASPackageKeys.PackageAttributesKeys.NAME:
                 package_name = package_name_attr[1]
@@ -1528,7 +1547,8 @@ def _get_installed_package_info(module, include_installed_files):
             for line in package_info_lines:
 
                 # split line by delimiter to get key and value
-                attr = str(line).split(delim)
+                line = _decode_str(line, locale_encoding)
+                attr = line.split(delim)
 
                 if len(attr) == 2:
                     key = attr[0]
@@ -1575,11 +1595,12 @@ def _get_installed_package_info(module, include_installed_files):
 # =====
 # _get_sas_package_update_info_yum(AnsibleModule)
 # =====
-def _get_sas_package_update_info_yum(module):
+def _get_sas_package_update_info_yum(module, locale_encoding=None):
     """
     Retrieves package information via yum for all installed SAS packages and returns the values as a dict.
 
     :param AnsibleModule module: The AnsibleModule object representing the current module.
+    :param locale_encoding: A custom locale encoding that should be used to decode potentially localized text.
     :return: A dict of package names mapped to a dict containing package attribute names and their values.
     :rtype dict:
     """
@@ -1602,14 +1623,16 @@ def _get_sas_package_update_info_yum(module):
     for line in all_update_info:
 
         # skip empty lines
-        if not str(line).strip().startswith('sas-'):
+        line = _decode_str(line, locale_encoding)
+        if not line.strip().startswith('sas-'):
             continue
 
         # replace multiple consecutive whitespaces delim character
         line = re.sub(' +', ':::', line)
 
         # split line by delimiter to get key and value
-        attr = str(line).split(':::')
+        line = _decode_str(line, locale_encoding)
+        attr = line.split(':::')
 
         if len(attr) == 3:
             # define a dictionary to map package update attributes to their respective key
@@ -1629,11 +1652,12 @@ def _get_sas_package_update_info_yum(module):
 # =====
 # _get_sas_package_update_info_zypper(AnsibleModule)
 # =====
-def _get_sas_package_update_info_zypper(module):
+def _get_sas_package_update_info_zypper(module, locale_encoding=None):
     """
     Retrieves package information via zypper for all installed SAS packages and returns the values as a dict.
 
     :param AnsibleModule module: The AnsibleModule object representing the current module.
+    :param locale_encoding: A custom locale encoding that should be used to decode potentially localized text.
     :return: A dict of package names mapped to a dict containing package attribute names and their values.
     :rtype dict:
     """
@@ -1658,7 +1682,8 @@ def _get_sas_package_update_info_zypper(module):
         line = re.sub(' +', ' ', line)
 
         # split line by delimiter to get key and value
-        attr = str(line).split(" | ")
+        line = _decode_str(line, locale_encoding)
+        attr = line.split(" | ")
 
         if len(attr) == 6:
             # define a dictionary to map package update attributes to their respective key
@@ -1784,6 +1809,32 @@ def _bytesHumanReadable(num_bytes, unit_step=1024.0):
     return str(num_bytes) + ' ' + unit
 
 
+# =====
+# Handle decoding strings that may contain encoded characters
+# =====
+def _decode_str(value, locale_encoding=None):
+    # no need to decode in Python3
+    if sys.version_info[0] >= 3:
+        return value
+
+    # encodings to try
+    encodings = ['ascii', 'utf-8', 'iso-8859-1']
+
+    for encoding in encodings:
+        try:
+            return value.decode(encoding).encode('utf-8')
+        except UnicodeDecodeError:
+            pass
+
+    # try the provided locale encoding if none of the default encodings worked
+    if locale_encoding is not None:
+        # if this raises an error, let it surface to make it known that the provided value wasn't accepted
+        return value.decode(locale_encoding).encode('utf-8')
+
+    # if no strict decode worked, decode to utf-8 and ignore unknown chars
+    return value.decode('utf-8', 'ignore').encode('utf-8')
+
+
 # =====
 # Script entry point
 # =====

playbooks/merge-playbook/library/merge_viya_deployment_files.py

@@ -166,17 +166,35 @@ def read_inventory(inv_file, comments):
             with open(tmp_file_name, 'w') as tmp_file:
                 tmp_file.writelines(i.replace('#', '#') for i in current_file.readlines())
 
-    inv_parser = configparser.ConfigParser(allow_no_value=True)
-    inv_parser.optionxform = lambda option: option  # preserve as case-sensitive
-    try:
-        inv_parser.read(tmp_file_name)
-    except configparser.MissingSectionHeaderError:
-        # need to inject [host-definitions] inventory section header to parse it
-        try_again = six.StringIO()
-        try_again.write('[host-definitions]\n')
-        try_again.write(open(tmp_file_name).read())
-        try_again.seek(0, os.SEEK_SET)
-        inv_parser.readfp(try_again)
+    # configparser compatability issues between python 2 and python 3 around strict
+    # parameter and DuplicateOptionError needs differential usage.   Changes were added
+    # to python 3.2.
+    if sys.version_info.major == 3 and sys.version_info.minor >= 2:  
+        # python version greater than or equal to 3.2
+        inv_parser = configparser.ConfigParser(strict=False, allow_no_value=True)
+        inv_parser.optionxform = lambda option: option  # preserve as case-sensitive
+        try:
+            inv_parser.read(tmp_file_name)
+        except (configparser.MissingSectionHeaderError, configparser.DuplicateOptionError):    
+            # need to inject [host-definitions] inventory section header to parse it
+            try_again = six.StringIO()
+            try_again.write('[host-definitions]\n')
+            try_again.write(open(tmp_file_name).read())
+            try_again.seek(0, os.SEEK_SET)
+            inv_parser.readfp(try_again)
+    else:
+        # python version less than 3.2
+        inv_parser = configparser.ConfigParser(allow_no_value=True)
+        inv_parser.optionxform = lambda option: option  # preserve as case-sensitive
+        try:
+            inv_parser.read(tmp_file_name)
+        except (configparser.MissingSectionHeaderError):
+            # need to inject [host-definitions] inventory section header to parse it
+            try_again = six.StringIO()
+            try_again.write('[host-definitions]\n')
+            try_again.write(open(tmp_file_name).read())
+            try_again.seek(0, os.SEEK_SET)
+            inv_parser.readfp(try_again)
 
     return inv_parser
 
@@ -521,7 +539,6 @@ def main():
         "merge_default_host": {"required": False, "type": "str"},
     }
     module = AnsibleModule(argument_spec=fields,
-                           check_invalid_arguments=True,
                            supports_check_mode=True)
 
     current_inventory_file = module.params['current_inventory_file']

playbooks/pre-install-playbook/roles/viya-ark.preinstall/defaults/main.yml

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2019-2020, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
+# Copyright (c) 2019-2021, SAS Institute Inc., Cary, NC, USA.  All Rights Reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 ---
@@ -12,6 +12,9 @@ os_major_version_1: 7
 os_major_minor_version_1: 7.1
 os_major_version_2: 6
 os_major_minor_version_2: 6.7
+os_major_version_3: 8
+os_major_minor_version_3: 8.0
+
 suse_os_major_version: 12
 suse_os_major_minor_version: 12.2
 
@@ -166,6 +169,11 @@ required_package_versions_rhel:
   - { name: curl, value: 7.19.7-53 }
   - { name: nss.x86_64, value: 3.36.0-7 }
 
+compat_packages:
+  - compat-openssl10
+  - authselect-compat
+  - ncurses-compat-libs
+
 ##
 ## SAS YUM repos.
 ##

playbooks/pre-install-playbook/roles/viya-ark.preinstall/tasks/pre.os_version_check.yml

@@ -22,7 +22,7 @@
         - ansible_os_family == redhat_os_name or ansible_os_family == suse_os_name 
       msg: |
         The OS of this server is not supported.
-        Viya requires {{redhat_os_name}} {{os_major_version_1}}, {{os_major_version_2}}, (at least {{os_major_minor_version_1}}, {{os_major_minor_version_2}} ) or {{suse_os_name}} at least version {{suse_os_major_minor_version}}
+        Viya requires {{redhat_os_name}} {{os_major_version_1}}, {{os_major_version_2}} or {{os_major_version_3}}, (at least {{os_major_minor_version_1}}, {{os_major_minor_version_2}} or {{os_major_minor_version_3}} ) or {{suse_os_name}} at least version {{suse_os_major_minor_version}}
         You seem to have {{ansible_os_family}} {{ansible_distribution_version}}
 
   - name: Ensure that the Operating System version is supported when on RedHat (this includes Oracle Linux)
@@ -32,17 +32,23 @@
         - ansible_distribution_major_version is version(os_major_version_1, '==')
           or
           ansible_distribution_major_version is version(os_major_version_2, '==')
-        # at least 6.7 if 6 or at least 7.1 if 7
+          or
+          ansible_distribution_major_version is version(os_major_version_3, '==')
+        # at least 6.7 if 6 or at least 7.1 if 7 or at least 8.0 if 8
         - (ansible_distribution_major_version is version(os_major_version_1, '==')
           and
           ansible_distribution_version is version(os_major_minor_version_1, '>='))
           or
           (ansible_distribution_major_version is version(os_major_version_2, '==')
           and
           ansible_distribution_version is version(os_major_minor_version_2, '>='))
+          or 
+          (ansible_distribution_major_version is version(os_major_version_3, '==')
+          and
+          ansible_distribution_version is version(os_major_minor_version_3, '>='))
       msg: |
         The OS of this server is not supported.
-        Viya requires {{redhat_os_name}} {{os_major_version_1}}, {{os_major_version_2}}, (at least {{os_major_minor_version_1}}, {{os_major_minor_version_2}} )
+        Viya requires {{redhat_os_name}} {{os_major_version_1}}, {{os_major_version_2}} or {{os_major_version_3}}, (at least {{os_major_minor_version_1}}, {{os_major_minor_version_2}} or {{os_major_minor_version_3}} )
         You seem to have {{ansible_os_family}} {{ansible_distribution_version}}
     when: ansible_os_family == redhat_os_name