Fix #206 and #201 Warn about GOSC DNS failures on Ubuntu and default gateway failures (#227)

* Fix #206 and #201 Warn about GOSC DNS failures on Ubuntu and default gateway failures

Signed-off-by: Qi Zhang <[email protected]>

Author: keirazhang

common/vm_add_serial_port.yml

@@ -9,7 +9,7 @@
 #   
 - name: "Set default serial port output file"
   set_fact:
-    vm_serial_port_file_path: "{{ vm_files_path_ds }}/serial-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}.log"
+    vm_serial_port_file_path: "{{ vm_files_path_ds.strip('\\/') }}/serial-{{ lookup('pipe', 'date +%Y%m%d%H%M%S') }}.log"
   when: vm_serial_port_file_path is undefined or not vm_serial_port_file_path
 
 - name: "Add a serial port using output file"

common/vm_remove_serial_port.yml

@@ -5,6 +5,14 @@
 # Parameter:
 #   vm_serial_port_file_path: The serial port output file on datastore.
 
+# Before removing serial port, remove the serial port output file
+- include_tasks: esxi_check_delete_datastore_file.yml
+  vars:
+    file_in_datastore: "{{ datastore }}"
+    file_in_datastore_path: "{{ vm_serial_port_file_path.split(']')[-1].strip(' ') }}"
+    file_in_datastore_ops: "absent"
+    file_in_datastore_failed_ignore: True
+
 - name: "Remove a serial port using output file"
   vmware_guest_serial_port:
     hostname: "{{ vsphere_host_name }}"

linux/deploy_vm/deploy_vm_from_iso.yml

@@ -172,6 +172,14 @@
       when:  guest_os_ansible_distribution == "Ubuntu"
 
     - include_tasks: ../../common/vm_remove_serial_port.yml
+
+    - fail:
+        msg: "Failed to remove serial port from VM"
+      when: >
+        remove_serial_port is undefined or
+        remove_serial_port.changed is undefined or
+        not remove_serial_port.changed
+
     - include_tasks: ../../common/vm_set_power_state.yml
       vars:
         vm_power_state_set: 'powered-on'

linux/guest_customization/check_dns.yml

@@ -44,14 +44,9 @@
             src_path: "/tmp/resolvectl_dns.txt"
             dest_path: "{{ current_test_log_folder }}/resolvectl_dns.txt"
 
-        - name: "Get output of 'resolvectl dns'"
-          command: cat "{{ current_test_log_folder }}/resolvectl_dns.txt"
-          register: dns_servers
-
         - name: "Set fact of guest DNS servers"
           set_fact:
-            guest_dns_servers: "{{ dns_servers.stdout_lines | select('match', '^Link') |
-                               regex_findall('\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b') }}"
+            guest_dns_servers: "{{ lookup('file', current_test_log_folder ~ '/resolvectl_dns.txt').split('\n') | select('match', '^link', ignorecase=True) | regex_findall('\\b(?:[0-9]{1,3}\\.){3}[0-9]{1,3}\\b') }}"
       when: resolvectl_dns_result | bool
 
     # Run command 'resolvectl domain' to get DNS servers
@@ -72,14 +67,9 @@
             src_path: "/tmp/resolvectl_domain.txt"
             dest_path: "{{ current_test_log_folder }}/resolvectl_domain.txt"
 
-        - name: "Get output of 'resolvectl domain'"
-          command: cat "{{ current_test_log_folder }}/resolvectl_domain.txt"
-          register: dns_domains
-
         - name: "Set fact of guest DNS searching domains"
           set_fact:
-            guest_dns_search: "{{ dns_domains.stdout_lines | select('match', '^Link') |
-                              regex_findall('\\b(?:[a-zA-Z0-9-]{1,}\\.){1,}[a-zA-Z0-9-]{1,}\\b') }}"
+            guest_dns_search: "{{ lookup('file', current_test_log_folder ~ '/resolvectl_domain.txt').split('\n') | select('match', '^link', ignorecase=True) | regex_findall('\\b(?:[a-zA-Z0-9-]{1,}\\.){1,}[a-zA-Z0-9-]{1,}\\b') }}"
       when: resolvectl_domain_result | bool
   when: resolvectl_status_result
 
@@ -101,27 +91,14 @@
         src_path: "/tmp/resolv.conf"
         dest_path: "{{ current_test_log_folder }}/etc/resolv.conf"
 
-    - name: "Get DNS servers from /etc/resolv.conf"
-      shell: cat "{{ current_test_log_folder }}/etc/resolv.conf" | grep -i "^nameserver\s*" | awk '{print $2}'
-      register: dns_servers
-
-    - name: "Get guest DNS servers from shell command output"
-      set_fact:
-        guest_dns_servers: "{{ dns_servers.stdout_lines }}"
-
-    - name: "Get DNS search domains from /etc/resolv.conf"
-      shell: cat "{{ current_test_log_folder }}/etc/resolv.conf" | grep -i "^search" | awk '{i=2;while(i<=NF) {print $i; i++ }}'
-      register: dns_search
-
-    - name: "Save guest DNS search list to tmp variable"
+    - name: "Get content of /etc/resolv.conf in guest OS"
       set_fact:
-        tmp_dns_search: "{{ dns_search.stdout_lines }}"
+        guest_resolv_conf: "{{ lookup('file', current_test_log_folder ~ '/etc/resolv.conf').split('\n') }}"
 
-    # In Amazon Linux 2 guest, a period after each DNS search domain would be added, so remove it
-    - name: "Get guest DNS search list"
+    - name: "Get guest DNS servers and search domains from /etc/resolv.conf"
       set_fact:
-        guest_dns_search: "{{ guest_dns_search + [item | regex_replace('\\.$','')] }}"
-      with_items: "{{ tmp_dns_search }}"
+        guest_dns_servers: "{{ guest_resolv_conf | select('match', '^nameserver\\s*') | map('regex_replace', 'nameserver\\s*', '') }}"
+        guest_dns_search: "{{ guest_resolv_conf | select('match', '^search\\s*') | join('\\n') | regex_replace('^search\\s*', '') | split() | map('regex_replace', '\\.$', '') }}"
   when:
     - not resolvectl_status_result
     - guest_os_ansible_distribution != 'Ubuntu'
@@ -143,21 +120,15 @@
             operation: "fetch_file"
             src_path: "/tmp/systemd_resolve_status"
             dest_path: "{{ current_test_log_folder }}/systemd_resolve_status.txt"
-        - name: "Get DNS servers from command systemd-resolve"
-          shell: cat "{{ current_test_log_folder }}/systemd_resolve_status.txt" | sed -n '/DNS Servers/,/DNS Domain/p' | head -n -1
-          register: dns_servers
-        - name: "Set fact of DNS servers from shell command output"
+
+        - name: "Set fact of guest systemd resolve status"
           set_fact:
-            guest_dns_servers: "{{ (guest_dns_servers + [item.strip().split(':')[1].strip()]) if 'DNS Servers' in item else (guest_dns_servers + [item.strip()]) }}"
-          with_items: "{{ dns_servers.stdout_lines }}"
+            guest_resolve_status: "{{ lookup('file', current_test_log_folder ~ '/systemd_resolve_status.txt') | regex_replace('\\n\\s*','\\n') }}"
 
-        - name: "Get DNS search domains from command systemd-resolve"
-          shell: cat "{{ current_test_log_folder }}/systemd_resolve_status.txt" | sed -n '/DNS Domain/,/DNSSEC NTA/p' | head -n -1
-          register: dns_domains
-        - name: "Set fact of DNS search domains from shell command output"
+        - name: "Set fact of DNS servers and search domains from systemd resolve status"
           set_fact:
-            guest_dns_search: "{{ (guest_dns_search + [item.strip().split(':')[1].strip()]) if 'DNS Domain' in item else (guest_dns_search + [item.strip()]) }}"
-          with_items: "{{ dns_domains.stdout_lines }}"
+            guest_dns_servers: "{{ guest_resolve_status | regex_search('DNS Servers:\\s*(([0-9]+\\.){3}([0-9]+)\\n?)+') | regex_replace('DNS Servers:\\s*', '') | split('\\n') | reject('equalto', '') }}"
+            guest_dns_search: "{{ guest_resolve_status | regex_search('DNS Domain:\\s*(([0-9a-zA-Z_\\-]+\\.){1,}([0-9a-zA-Z_\\-]+)\\n?)+') | regex_replace('DNS Domain:\\s*', '') | split('\\n') | reject('equalto', '') }}" 
       when: systemd_resolve_result
   when:
     - not resolvectl_status_result
@@ -207,6 +178,11 @@
     gosc_check_error_msg: "VM DNS servers are {{ guest_dns_servers }}, not expected DNS servers {{ linux_gosc_spec['dns_servers'] }}"
 
 # Check guest DNS search domains
+- debug:
+    msg:
+      - "The expected search domains: {{ linux_gosc_spec['dns_suffix'] }}"
+      - "The actual DNS search domains got in guest: {{ guest_dns_search }}"
+
 - name: "Check DNS search domains are customized"
   set_fact:
     dns_suffix_success: "{{ linux_gosc_spec['dns_suffix'] | difference(guest_dns_search) | length == 0 }}"

linux/guest_customization/check_etc_hosts.yml

@@ -9,6 +9,7 @@
     etc_hosts_success: False
     expected_ips: ['::1']
     ipv6_hostname_check: False
+    actual_ips: []
 
 - include_tasks: ../../common/vm_guest_file_operation.yml
   vars:
@@ -17,12 +18,13 @@
     dest_path: "{{ hosts_file_path }}"
 
 - name: "Get IP bindings for hostname {{ linux_gosc_spec['hostname'] }} in /etc/hosts"
-  shell: grep "{{ linux_gosc_spec['hostname'] }}.{{ linux_gosc_spec['domain'] }} *{{ linux_gosc_spec['hostname'] }}" {{ hosts_file_path }} | awk '{print $1}'
-  changed_when: False
-  register: etc_hosts_result
-- name: "Save hostname binding list"
   set_fact:
-    actual_ips: "{{ etc_hosts_result.stdout_lines }}"
+    guest_host_ip_bindings: "{{ lookup('file', hosts_file_path).split('\n') | select('match', '.*\\s*' ~ linux_gosc_spec['hostname'] ~ '.' ~ linux_gosc_spec['domain'] ~ ' *' ~ linux_gosc_spec['hostname']) }}"
+
+- name: "Set fact of hostnam and IP binding"
+  set_fact:
+    actual_ips: "{{ guest_host_ip_bindings | map('regex_replace', '\\s*' ~ linux_gosc_spec['hostname'] ~ '.*', '') }}"
+  when: guest_host_ip_bindings | length > 0
 
 # Perl GOSC:
 #   1. When ip assignment is dhcp, hostname will be bound to 127.0.1.1 and ::1

linux/guest_customization/check_hostname_and_domain.yml

@@ -3,6 +3,8 @@
 ---
 - name: "Initialize hostname and domain check result"
   set_fact:
+    guest_hostname: ""
+    guest_dns_domain: ""
     hostname_success: ""
     dns_domain_success: ""
 
@@ -27,14 +29,13 @@
     dest_path: "{{ current_test_log_folder }}/hostname_f.txt"
 
 - name: "Get guest FQDN"
-  command: cat "{{ current_test_log_folder }}/hostname_f.txt"
-  changed_when: False
-  register: result
+  set_fact:
+    guest_fqdn: "{{ lookup('file', current_test_log_folder ~ '/hostname_f.txt') }}"
 
-- name: "Save hostname and domain check result"
+- name: "Set fact of guest OS hostname and domain"
   set_fact:
-    guest_hostname: "{{ result.stdout.split('.')[0] }}"
-    guest_dns_domain: "{{ result.stdout.split('.')[1:] | join('.') }}"
+    guest_hostname: "{{ guest_fqdn.split('.')[0] }}"
+    guest_dns_domain: "{{ (guest_fqdn.split('.')[1:] | join('.')) if guest_fqdn.split('.') | length > 1 else '' }}"
 
 # Check hostname
 - name: Check guest hostname is customized

linux/guest_customization/check_network_config.yml

@@ -3,15 +3,14 @@
 ---
 - name: "Initialize GOSC network check result"
   set_fact:
+    ip_command: "{{ '/usr/sbin/ip' if guest_os_ansible_distribution != 'Ubuntu' else '/sbin/ip' }}"
+    guest_iface_name: ""
+    guest_ipv4_address: ""
+    guest_ipv4_netmask: ""
+    guest_ipv4_gateway: ""
     guest_ipv4_success: ""
     guest_netmask_success: ""
     guest_gateway_success: ""
-    ip_command: "/usr/sbin/ip"
-
-- name: "Set fact of ip command when guest OS is Ubuntu"
-  set_fact:
-    ip_command: "/sbin/ip"
-  when: guest_os_ansible_distribution in ["Ubuntu"]
 
 # Get IP address and netmask
 - include_tasks: ../../common/vm_shell_in_guest.yml
@@ -27,15 +26,15 @@
     dest_path: "{{ current_test_log_folder }}/ip_addr_show.txt"
 
 - name: "Get IPv4 address"
-  shell: cat "{{ current_test_log_folder }}/ip_addr_show.txt" | grep -v lo | grep -v virbr | awk '{print $1, $2, $3}'
-  changed_when: False
-  register: result
+  set_fact:
+    guest_ip_addr_show: "{{ lookup('file', current_test_log_folder ~ '/ip_addr_show.txt').split('\n') | select('search', '^e(n|th).*') }}"
 
-- name: "Save GOSC network facts from command output"
+- name: "Set facts of guest network interface names and IPv4 address"
   set_fact:
-    guest_iface_name: "{{ result.stdout.split(' ')[0] }}"
-    guest_ipv4_address: "{{ result.stdout.split(' ')[-1] | ipaddr('address') }}"
-    guest_ipv4_netmask: "{{ result.stdout.split(' ')[-1] | ipaddr('netmask') }}"
+    guest_iface_name: "{{ guest_ip_addr_show[0].split()[0] }}"
+    guest_ipv4_address: "{{ guest_ip_addr_show[0] | regex_search('(\\d+\\.){3}(\\d+)/(\\d+)') | ipaddr('address') }}"
+    guest_ipv4_netmask: "{{ guest_ip_addr_show[0] | regex_search('(\\d+\\.){3}(\\d+)/(\\d+)') | ipaddr('netmask') }}"
+  when: guest_ip_addr_show | length > 0
 
 # Get default gateway
 - include_tasks: ../../common/vm_shell_in_guest.yml
@@ -51,12 +50,15 @@
     dest_path: "{{ current_test_log_folder }}/ip_route_show.txt"
 
 - name: "Get default gateway"
-  shell: cat "{{ current_test_log_folder }}/ip_route_show.txt" | grep "^default"
-  changed_when: False
-  register: result
-- name: "Save GOSC network gateway"
   set_fact:
-    guest_ipv4_gateway: "{{ result.stdout.split(' ')[2] }}"
+    guest_default_gateway: "{{ lookup('file', current_test_log_folder ~ '/ip_route_show.txt').split('\n') | select('match', '^default') }}"
+
+- name: "Set fact of GOSC network gateway"
+  set_fact:
+    guest_ipv4_gateway: "{{ guest_default_gateway[0].split()[2] }}"
+  when:
+    - guest_default_gateway | length > 0
+    - guest_default_gateway[0].split() | length >= 3
 
 # Check IP address, netmaks, gateway settings for GOSC with DHCP IP
 - block:
@@ -186,15 +188,15 @@
         dest_path: "{{ current_test_log_folder }}/list_network_files.txt"
 
     - name: "Look for network file on Photon OS"
-      command: cat "{{ current_test_log_folder }}/list_network_files.txt"
-      register: network_files
-
+      set_fact:
+        guest_network_files: "{{ lookup('file', current_test_log_folder ~ '/list_network_files.txt').split('\n') }}"
     - include_tasks: ../../common/vm_guest_file_operation.yml
       vars:
         operation: "fetch_file"
         src_path: "{{ src_network_file }}"
         dest_path: "{{ current_test_log_folder }}{{ src_network_file }}"
-      with_items: "{{ network_files.stdout_lines }}"
+      with_items: "{{ guest_network_files }}"
       loop_control:
         loop_var: "src_network_file"
+      when: guest_network_files | length > 0
   when: guest_os_ansible_distribution == "VMware Photon OS"

linux/guest_customization/check_timezone_and_hwclock.yml

@@ -3,9 +3,12 @@
 ---
 - name: "Initialize timezone check result"
   set_fact:
+    guest_timezone: ""
     timezone_success: ""
+
 - name: "Initialize hardware clock check result for perl GOSC"
   set_fact:
+    guest_hwclockUTC: ""
     hwclock_success: ""
   when: gosc_workflow == "perl"
 
@@ -25,12 +28,12 @@
         dest_path: "{{ current_test_log_folder }}/timedatectl_status.txt"
 
     - name: "Get time zone from 'timedatectl status'"
-      shell: grep -i "^ *time zone:" "{{ current_test_log_folder }}/timedatectl_status.txt" | awk '{print $3}'
-      register: tz_result
+      set_fact:
+        guest_timedate_status: "{{ lookup('file', current_test_log_folder ~ '/timedatectl_status.txt') | regex_search('\\s*Time zone:\\s*[^\\n]+\\n?') }}"
 
-    - name: "Save guest timezone setting after GOSC"
+    - name: "Set fact of guest time zone after GOSC"
       set_fact:
-        guest_timezone: "{{ tz_result.stdout }}"
+        guest_timezone: "{{ guest_timedate_status | regex_replace('.*\\s*Time zone:\\s*([^\\s]+)\\s*.*\\n?', '\\1')}}"
 
     - block:
         - name: "Get hwclock from 'timedatectl status'"
@@ -61,13 +64,8 @@
         dest_path: "{{ current_test_log_folder }}/readlink_etc_localtime.txt"
 
     - name: "Get timezone info from /etc/localtime"
-      command: cat "{{ current_test_log_folder }}/readlink_etc_localtime.txt"
-      register: tz_result
-      changed_when: False
-
-    - name: "Save guest timezone setting after GOSC"
       set_fact:
-        guest_timezone: "{{ tz_result.stdout.replace('/usr/share/zoneinfo/','') }}"
+        guest_timezone: "{{ lookup('file', current_test_log_folder ~ '/readlink_etc_localtime.txt') | replace('/usr/share/zoneinfo/','') }}"
 
     # Get hwclock by running command "hwclock -D" when test perl GOSC
     - block:

linux/guest_customization/linux_gosc_start.yml

@@ -44,6 +44,7 @@
 # Do not fail GOSC immediately if state keyword doesn't exist
 - include_tasks: ../../common/vm_wait_gosc_completed.yml
   vars:
+    get_guest_reset_time_retries: 10
     check_gosc_state_keyword: False
 
 # Traditional GOSC will reboot guest OS (except for Photon) after all customization done