PCI PT Hotplug/Hotunplug fixes for arch ppc64le (autotest#6072)

1) Added 2 new test cases :
   test 1 :  hotplog/unplug operation is performed for multiple times
   test 2 :  multiple hotplug/unplug operations with random reboot is done in between to check device avialability inside VM.
2) Changes made are :
   1) two new parameters added into cfg file
      multiple_hotplug_hotunplug
      number_of_hotplug_unplug
   2) params.itervalues() [supported in python2] was removed in python3. Python3 has params.values() instead. Hence made this change as it was failing to read the params with .itervalues()
   3) The "librtas" package name varies across different distros (fedora, sles etc). Therefore, the test first checks the running distro and adds the corresponding package name to the package list. This approach ensures seamless testing across all supported distributions.
   4) Zip() is the correct and Python 3-compatible way to iterate over two lists in parallel.
   5) For PP64LE arch in latest libvirt it is necessary to detach the device from the host before attaching it to a VM. Hence added a arch 'if' flag to check if the arch is ppc64le and libvirt is latest newer from 3.10 then detach the device from host first then attach it to VM.
   6) utils_net.ping is same, no changes made to the ping function, just added timeout=30 sec and passing session handler to the same ping function, so that ping command is executed inside VM , as my requirement is to run ping test inside VM for the interface added for a pci device.
   7) rand_reboot function is added. The function is designed where it Set probability of reboot per iteration (e.g., 30%) and based on total number of iterations and probability a random reboot happens in between hotplug/unplug operations. On an average for 30 iterations 6-8 reboots are expected.
   8) Both the tests are called inside try block

Signed-off-by: Tasmiya Nalatwad <tasmiya@linux.vnet.ibm.com>

Author: TasmiyaNalatwad

libvirt/tests/cfg/passthrough/pci/libvirt_pci_passthrough_hotplug.cfg

@@ -45,3 +45,9 @@
         - virsh_dump:
             # dump the guest with memory only option
             virsh_dump = "yes"
+        - multiple_hotplug_hotunplug:
+            multiple_hotplug_hotunplug = "yes"
+            number_of_hotplug_unplug = 25
+        - multiple_hotplug_unplug_with_rand_reboot:
+            random_reboot = "yes"
+            number_of_hotplug_unplug = 30

libvirt/tests/src/passthrough/pci/libvirt_pci_passthrough_hotplug.py

@@ -10,25 +10,28 @@
 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
 #
 # Author: Prudhvi Miryala<mprudhvi@linux.vnet.ibm.com>
+# Author: Tasmiya.Nalatwad<tasmiya@linux.vnet.ibm.com>
 # Host device hotplug test
 
 
 import os
 import logging as log
 import aexpect
 import time
+import platform
+import random
 
 from avocado.utils import process
+from avocado.utils import distro
 
 from virttest import virsh
 from virttest.libvirt_xml.vm_xml import VMXML
 from virttest.libvirt_xml.nodedev_xml import NodedevXML
 from virttest.test_setup import PciAssignable
-from virttest import utils_misc
+from virttest import utils_test, utils_misc
 from virttest import data_dir
 from virttest.libvirt_xml.devices.controller import Controller
 from virttest import utils_package
-from virttest import utils_net
 from virttest import libvirt_version
 from virttest.utils_test import libvirt
 
@@ -56,6 +59,9 @@ def run(test, params, env):
         10. test virsh dumpxml
         11. hotunplug the device
         12. test stress
+        13. Perform multiple hotplug/unplug operations
+        14. Perfomr random reboots in between
+           multiple hotplug/unplug
            to verify the new network device.
     """
     # get the params from params
@@ -75,8 +81,16 @@ def run(test, params, env):
     virsh_dumpxml = params.get("virsh_dumpxml", "no")
     virsh_dump = params.get("virsh_dump", "no")
     flood_ping = params.get("flood_ping", "no")
+    arch = platform.machine()
+    multiple_hotplug_hotunplug = params.get("multiple_hotplug_hotunplug", "no")
+    number_of_hotplug_unplug = int(params.get("number_of_hotplug_unplug", "1"))
+    random_reboot = params.get("random_reboot", "no")
+    cntlr_index = params.get("index", "1")
+    cntlr_model = params.get("model", "pci-root")
+    cntlr_type = "pci"
+
     # Check the parameters from configuration file.
-    for each_param in params.itervalues():
+    for each_param in params.values():
         if "ENTER_YOUR" in each_param:
             test.cancel("Please enter the configuration details of %s."
                         % each_param)
@@ -85,19 +99,37 @@ def run(test, params, env):
     devices = vmxml.get_devices()
     pci_devs = []
     dargs = {'debug': True, 'ignore_status': True}
+
+    controllers = vmxml.get_controllers(cntlr_type, cntlr_model)
+    index_list = []
+    for controller in controllers:
+        index_value = controller.get("index")
+        if index_value is not None:
+            index_list.append(int(index_value))
+
+    if index_list:
+        next_index = max(index_list) + 1
+    else:
+        next_index = int(cntlr_index)
     controller = Controller("controller")
-    controller.type = "pci"
-    controller.index = params.get("index", "1")
-    controller.model = params.get("model", "pci-root")
+    controller.type = cntlr_type
+    controller.index = str(next_index)
+    controller.model = cntlr_model
+
     devices.append(controller)
     vmxml.set_devices(devices)
     vmxml.sync()
     if not vm.is_alive():
         vm.start()
         session = vm.wait_for_login()
-    if not utils_package.package_install(["ppc64-diag",
-                                          "librtas", "powerpc-utils"],
-                                         session, 360):
+    detected_distro = distro.detect()
+    pkg = ["ppc64-diag", "powerpc-utils"]
+    if detected_distro.name == "sles16":
+        if int(detected_distro.version) == 16:
+            pkg.extend(["librtas2"])
+    if detected_distro.name in ("fedora", "rhel"):
+        pkg.extend(["librtas"])
+    if not utils_package.package_install(pkg, session, 360):
         test.cancel('Fail on dependencies installing')
     if virsh_dump == "yes":
         dump_file = os.path.join(data_dir.get_tmp_dir(), "virshdump.xml")
@@ -122,7 +154,7 @@ def run(test, params, env):
 
     def detach_device(pci_devs, pci_ids):
         # detaching the device from host
-        for pci_value, pci_node in map(None, pci_devs, pci_ids):
+        for pci_value, pci_node in zip(pci_devs, pci_ids):
             pci_value = pci_value.replace(".", "_")
             cmd = "lspci -ks %s | grep 'Kernel driver in use' |\
                    awk '{print $5}'" % pci_node
@@ -138,7 +170,7 @@ def detach_device(pci_devs, pci_ids):
 
     def reattach_device(pci_devs, pci_ids):
         # reattach the device to host
-        for pci_value, pci_node in map(None, pci_devs, pci_ids):
+        for pci_value, pci_node in zip(pci_devs, pci_ids):
             pci_value = pci_value.replace(".", "_")
             cmd = "lspci -ks %s | grep 'Kernel driver in use' |\
                    awk '{print $5}'" % pci_node
@@ -160,8 +192,12 @@ def check_attach_pci():
         return nic_list_after != nic_list_before
 
     def device_hotplug():
-        if not libvirt_version.version_compare(3, 10, 0):
-            detach_device(pci_devs, pci_ids)
+        if arch == "ppc64le":
+            if libvirt_version.version_compare(3, 10, 0):
+                detach_device(pci_devs, pci_ids)
+        else:
+            if not libvirt_version.version_compare(3, 10, 0):
+                detach_device(pci_devs, pci_ids)
         # attach the device in hotplug mode
         result = virsh.attach_device(vm_name, dev.xml,
                                      flagstr="--live", debug=True)
@@ -190,16 +226,18 @@ def device_hotunplug():
 
     def test_ping():
         try:
-            output = session.cmd_output("lspci -nn | grep %s" % device_name)
+            session = vm.wait_for_login()
+            output = session.cmd_output('lspci -nn | grep "%s"' % device_name)
             nic_id = str(output).split(' ', 1)[0]
             nic_name = str(utils_misc.get_interface_from_pci_id(nic_id,
                                                                 session))
             session.cmd("ip addr flush dev %s" % nic_name)
             session.cmd("ip addr add %s/%s dev %s"
                         % (net_ip, netmask, nic_name))
             session.cmd("ip link set %s up" % nic_name)
-            s_ping, o_ping = utils_net.ping(dest=server_ip, count=5,
-                                            interface=net_ip)
+            s_ping, o_ping = utils_test.ping(dest=server_ip, count=5,
+                                             interface=net_ip, timeout=30,
+                                             session=session)
             logging.info(s_ping)
             logging.info(o_ping)
             if s_ping:
@@ -211,8 +249,9 @@ def test_ping():
 
     def test_flood_ping():
         # Test Flood Ping
-        s_ping, o_ping = utils_net.ping(dest=server_ip, count=5,
-                                        interface=net_ip, flood=True)
+        s_ping, o_ping = utils_test.ping(dest=server_ip, count=5,
+                                         interface=net_ip, timeout=30,
+                                         flood=True, session=session)
         logging.info(s_ping)
         logging.info(o_ping)
         if s_ping:
@@ -263,6 +302,15 @@ def test_dump():
         if cmd_result.exit_status:
             test.fail("Failed to virsh dump of domain %s" % vm_name)
 
+    def rand_reboot(iteration):
+        #  Set probability of reboot per iteration (e.g., 30%)
+        reboot_chance = 0.3
+        if random.random() < reboot_chance:
+            logging.debug("Rebooting at iteration %s", iteration)
+            test_reboot()
+            time.sleep(5)
+            logging.debug("Random reboot completed, and adapter found in the VM after reboot")
+
     try:
         for stress_value in range(0, int(stress_val)):
             device_hotplug()
@@ -279,6 +327,21 @@ def test_dump():
                 test_dump()
             device_hotunplug()
 
+        if multiple_hotplug_hotunplug == "yes":
+            for iteration in range(number_of_hotplug_unplug):
+                logging.info("Performing Hotplug/Hotunplug of pci device for %s time", iteration)
+                device_hotplug()
+                test_ping()
+                device_hotunplug()
+
+        if random_reboot == "yes":
+            for iteration in range(number_of_hotplug_unplug):
+                logging.info("Performing Hotplug/Hotunplug of pci device for %s time", iteration)
+                device_hotplug()
+                test_ping()
+                rand_reboot(iteration)
+                device_hotunplug()
+
     finally:
         # clean up
         data_dir.clean_tmp_files()