diff --git a/README.md b/README.md index 497771d8..97c1d7ef 100644 --- a/README.md +++ b/README.md @@ -3,9 +3,10 @@ Note: this is a perpertual work in progress. If you encounter any obstacles or bugs, let us know! ## Main requirements -* python >= 3.5 -* pytest >= 5.4 (python3 version) -* xo-cli >= 0.17.0 installed, in the PATH, and registered to an instance of XO that will be used during the tests +* python >= 3.8 +* packages as listed in requirements/base.txt +* extra test-specific requirements are documented in the test file + "Requirements" header ### Quick install (python requirements) @@ -13,14 +14,12 @@ Install the python requirements using pip: ``` $ pip install -r requirements/base.txt - ``` Additionally, for dev dependencies (things like the linter / style checker): ``` $ pip install -r requirements/dev.txt - ``` ## Other requirements diff --git a/conftest.py b/conftest.py index 1e6c0bfc..66f2ca14 100644 --- a/conftest.py +++ b/conftest.py @@ -1,5 +1,6 @@ import itertools import logging +import os import pytest import tempfile @@ -7,11 +8,14 @@ import lib.config as global_config +from lib import pxe +from lib.common import callable_marker, shortened_nodeid from lib.common import wait_for, vm_image, is_uuid from lib.common import setup_formatted_and_mounted_disk, teardown_formatted_and_mounted_disk from lib.netutil import is_ipv6 from lib.pool import Pool -from lib.vm import VM +from lib.sr import SR +from lib.vm import VM, xva_name_from_def from lib.xo import xo_cli # Import package-scoped fixtures. Although we need to define them in a separate file so that we can @@ -19,22 +23,22 @@ # need to import them in the global conftest.py so that they are recognized as fixtures. from pkgfixtures import formatted_and_mounted_ext4_disk, sr_disk_wiped -# *** Support for incremental tests in test classes *** -# From https://stackoverflow.com/questions/12411431/how-to-skip-the-rest-of-tests-in-the-class-if-one-has-failed -def pytest_runtest_makereport(item, call): - if "incremental" in item.keywords: - if call.excinfo is not None: - parent = item.parent - parent._previousfailed = item - -def pytest_runtest_setup(item): - previousfailed = getattr(item.parent, "_previousfailed", None) - if previousfailed is not None: - pytest.skip("previous test failed (%s)" % previousfailed.name) +# Do we cache VMs? +try: + from data import CACHE_IMPORTED_VM +except ImportError: + CACHE_IMPORTED_VM = False +assert CACHE_IMPORTED_VM in [True, False] -# *** End of: Support for incremental tests *** +# pytest hooks def pytest_addoption(parser): + parser.addoption( + "--nest", + action="store", + default=None, + help="XCP-ng or XS master of pool to use for nesting hosts under test", + ) parser.addoption( "--hosts", action="append", @@ -85,9 +89,97 @@ def pytest_configure(config): global_config.ignore_ssh_banner = config.getoption('--ignore-ssh-banner') global_config.ssh_output_max_lines = int(config.getoption('--ssh-output-max-lines')) -def setup_host(hostname_or_ip): +def pytest_generate_tests(metafunc): + if "vm_ref" in metafunc.fixturenames: + vms = metafunc.config.getoption("vm") + if not vms: + vms = [None] # no --vm parameter does not mean skip the test, for us, it means use the default + metafunc.parametrize("vm_ref", vms, indirect=True, scope="module") + +def pytest_collection_modifyitems(items, config): + # Automatically mark tests based on fixtures they require. + # Check pytest.ini or pytest --markers for marker descriptions. + + markable_fixtures = [ + 'uefi_vm', + 'unix_vm', + 'windows_vm', + 'hostA2', + 'hostB1', + 'sr_disk', + 'sr_disk_4k' + ] + + for item in items: + fixturenames = getattr(item, 'fixturenames', ()) + for fixturename in markable_fixtures: + if fixturename in fixturenames: + item.add_marker(fixturename) + + if 'vm_ref' not in fixturenames: + item.add_marker('no_vm') + + if item.get_closest_marker('multi_vms'): + # multi_vms implies small_vm + item.add_marker('small_vm') + +# BEGIN make test results visible from fixtures +# from https://docs.pytest.org/en/latest/example/simple.html#making-test-result-information-available-in-fixtures + +# FIXME we may have to move this into lib/ if fixtures in sub-packages +# want to make use of this feature + +PHASE_REPORT_KEY = pytest.StashKey[dict[str, pytest.CollectReport]]() +@pytest.hookimpl(wrapper=True, tryfirst=True) +def pytest_runtest_makereport(item, call): + # execute all other hooks to obtain the report object + rep = yield + + # store test results for each phase of a call, which can + # be "setup", "call", "teardown" + item.stash.setdefault(PHASE_REPORT_KEY, {})[rep.when] = rep + + return rep + +# END make test results visible from fixtures + + +# fixtures + +def setup_host(hostname_or_ip, *, config=None): + host_vm = None + if hostname_or_ip.startswith("cache://"): + nest_hostname = config.getoption("nest") + if not nest_hostname: + pytest.fail("--hosts=cache://... requires --nest parameter") + nest = Pool(nest_hostname).master + + protocol, rest = hostname_or_ip.split(":", 1) + host_vm = nest.import_vm(f"clone+start:{rest}", nest.main_sr_uuid(), + use_cache=CACHE_IMPORTED_VM) + + vif = host_vm.vifs()[0] + mac_address = vif.param_get('MAC') + logging.info("Nested host has MAC %s", mac_address) + + # catch host-vm IP address + wait_for(lambda: pxe.arp_addresses_for(mac_address), + "Wait for DHCP server to see nested host in ARP tables", + timeout_secs=10 * 60) + ips = pxe.arp_addresses_for(mac_address) + logging.info("Nested host has IPs %s", ips) + assert len(ips) == 1 + host_vm.ip = ips[0] + + wait_for(lambda: not os.system(f"nc -zw5 {host_vm.ip} 22"), + "Wait for ssh up on nested host", retry_delay_secs=5) + + hostname_or_ip = host_vm.ip + pool = Pool(hostname_or_ip) h = pool.master + if host_vm: + h.nested = host_vm return h @pytest.fixture(scope='session') @@ -96,11 +188,17 @@ def hosts(pytestconfig): hosts_args = pytestconfig.getoption("hosts") hosts_split = [hostlist.split(',') for hostlist in hosts_args] hostname_list = list(itertools.chain(*hosts_split)) - host_list = [setup_host(hostname_or_ip) for hostname_or_ip in hostname_list] + host_list = [setup_host(hostname_or_ip, config=pytestconfig) + for hostname_or_ip in hostname_list] if not host_list: pytest.fail("This test requires at least one --hosts parameter") yield host_list + for host in host_list: + if host.nested: + logging.info("Destroying nested host's VM %s", host.nested) + host.nested.destroy(verify=True) + @pytest.fixture(scope='session') def registered_xo_cli(): # The fixture is not responsible for establishing the connection. @@ -302,19 +400,12 @@ def vm_ref(request): @pytest.fixture(scope="module") def imported_vm(host, vm_ref): - # Do we cache VMs? - try: - from data import CACHE_IMPORTED_VM - except ImportError: - CACHE_IMPORTED_VM = False - assert CACHE_IMPORTED_VM in [True, False] - if is_uuid(vm_ref): vm_orig = VM(vm_ref, host) name = vm_orig.name() logging.info(">> Reuse VM %s (%s) on host %s" % (vm_ref, name, host)) else: - vm_orig = host.import_vm(vm_ref, host.main_sr(), use_cache=CACHE_IMPORTED_VM) + vm_orig = host.import_vm(vm_ref, host.main_sr_uuid(), use_cache=CACHE_IMPORTED_VM) if CACHE_IMPORTED_VM: # Clone the VM before running tests, so that the original VM remains untouched @@ -331,19 +422,174 @@ def imported_vm(host, vm_ref): logging.info("<< Destroy VM") vm.destroy(verify=True) +@pytest.fixture(scope="function") +def create_vms(request, host): + """ + Returns list of VM objects created from `vm_definitions` marker. + + `vm_definitions` marker test author to specify one or more VMs, + using one `dict` per VM. + + Mandatory keys: + - `name`: name of the VM to create (str) + - `template`: name (or UUID) of template to use (str) + + Optional keys: see example below + + Example: + ------- + > @pytest.mark.vm_definitions( + > dict(name="vm1", template="Other install media"), + > dict(name="vm2", + > template="CentOS 7", + > params=( + > dict(param_name="memory-static-max", value="4GiB"), + > dict(param_name="HVM-boot-params", key="order", value="dcn"), + > ), + > vdis=[dict(name="vm 2 system disk", + > size="100GiB", + > device="xvda", + > userdevice="0", + > )], + > cd_vbd=dict(device="xvdd", userdevice="3"), + > vifs=[dict(index=0, network_uuid=NETWORKS["MGMT"])], + > )) + > def test_foo(create_vms): + > ... + + Example: + ------- + > @pytest.mark.dependency(depends=["test_foo"]) + > @pytest.mark.vm_definitions(dict(name="vm1", image_test="test_foo", image_vm="vm2")) + > def test_bar(create_vms): + > ... + + """ + import git + test_repo = git.Repo(".") + assert not test_repo.is_dirty(), "test repo must not be dirty to cache images" + + marker = request.node.get_closest_marker("vm_definitions") + if marker is None: + raise Exception("No vm_definitions marker specified.") + param_mapping = marker.kwargs.get("param_mapping", {}) + + vm_defs = [] + for vm_def in marker.args: + vm_def = callable_marker(vm_def, request, param_mapping=param_mapping) + assert "name" in vm_def + assert "template" in vm_def or "image_test" in vm_def + if "template" in vm_def: + assert "image_test" not in vm_def + # FIXME should check optional vdis contents + # FIXME should check for extra args + vm_defs.append(vm_def) + + try: + vms = [] + vdis = [] + vbds = [] + for vm_def in vm_defs: + if "template" in vm_def: + _create_vm(vm_def, host, vms, vdis, vbds) + elif "image_test" in vm_def: + _import_vm(request, vm_def, host, vms, test_repo, use_cache=CACHE_IMPORTED_VM) + yield vms + + # request.node is an "item" because this fixture has "function" scope + report = request.node.stash[PHASE_REPORT_KEY] + if report["setup"].failed: + logging.warning("setting up a test failed or skipped: not exporting VMs") + elif ("call" not in report) or report["call"].failed: + logging.warning("executing test failed or skipped: not exporting VMs") + else: + # record this state + for vm_def, vm in zip(vm_defs, vms): + # FIXME where to store? + gitref = test_repo.head.commit.hexsha + xva_name = f"{shortened_nodeid(request.node.nodeid)}-{vm_def['name']}-{gitref}.xva" + host.ssh(["rm -f", xva_name]) + vm.export(xva_name, "zstd", use_cache=CACHE_IMPORTED_VM) + + except Exception: + logging.error("exception caught...") + raise + + finally: + for vbd in vbds: + logging.info("<< Destroy VBD %s", vbd.uuid) + vbd.destroy() + for vdi in vdis: + logging.info("<< Destroy VDI %s", vdi.uuid) + vdi.destroy() + for vm in vms: + logging.info("<< Destroy VM %s", vm.uuid) + vm.destroy(verify=True) + +def _create_vm(vm_def, host, vms, vdis, vbds): + vm_name = vm_def["name"] + vm_template = vm_def["template"] + + logging.info(">> Install VM %r from template %r", vm_name, vm_template) + + vm = host.vm_from_template(vm_name, vm_template) + + # VM is now created, make sure we clean it up on any subsequent failure + vms.append(vm) + + if "vdis" in vm_def: + for vdi_def in vm_def["vdis"]: + sr = SR(host.main_sr_uuid(), host.pool) + vdi = sr.create_vdi(vdi_def["name"], vdi_def["size"]) + vdis.append(vdi) + # connect to VM + vbd = vm.create_vbd(vdi_def["device"], vdi.uuid) + vbds.append(vbd) + vbd.param_set(param_name="userdevice", value=vdi_def["userdevice"]) + + if "cd_vbd" in vm_def: + vm.create_cd_vbd(**vm_def["cd_vbd"]) + + if "vifs" in vm_def: + for vif_def in vm_def["vifs"]: + vm.create_vif(vif_def["index"], vif_def["network_uuid"]) + + if "params" in vm_def: + for param_def in vm_def["params"]: + logging.info("Setting param %s", param_def) + vm.param_set(**param_def) + +def _import_vm(request, vm_def, host, vms, test_repo, *, use_cache): + vm_image = xva_name_from_def(vm_def, request.node.nodeid, test_repo.head.commit.hexsha) + base_vm = host.import_vm(vm_image, sr_uuid=host.main_sr_uuid(), use_cache=use_cache) + + if use_cache: + # Clone the VM before running tests, so that the original VM remains untouched + logging.info(">> Clone cached VM before running tests") + vm = base_vm.clone() + # Remove the description, which may contain a cache identifier + vm.param_set('name-description', "") + else: + vm = base_vm + vms.append(vm) + @pytest.fixture(scope="module") -def running_vm(imported_vm): +def started_vm(imported_vm): vm = imported_vm - # may be already running if we skipped the import to use an existing VM if not vm.is_running(): vm.start() wait_for(vm.is_running, '> Wait for VM running') - wait_for(vm.try_get_and_store_ip, "> Wait for VM IP") - wait_for(vm.is_ssh_up, "> Wait for VM SSH up") + wait_for(vm.try_get_and_store_ip, "> Wait for VM IP", timeout_secs=5 * 60) return vm # no teardown +@pytest.fixture(scope="module") +def running_vm(started_vm): + vm = started_vm + wait_for(vm.is_ssh_up, "> Wait for VM SSH up") + return vm + @pytest.fixture(scope='module') def unix_vm(imported_vm): vm = imported_vm @@ -415,37 +661,3 @@ def second_network(pytestconfig, host): if network_uuid == host.management_network(): pytest.fail("--second-network must NOT be the management network") return network_uuid - -def pytest_generate_tests(metafunc): - if "vm_ref" in metafunc.fixturenames: - vms = metafunc.config.getoption("vm") - if not vms: - vms = [None] # no --vm parameter does not mean skip the test, for us, it means use the default - metafunc.parametrize("vm_ref", vms, indirect=True, scope="module") - -def pytest_collection_modifyitems(items, config): - # Automatically mark tests based on fixtures they require. - # Check pytest.ini or pytest --markers for marker descriptions. - - markable_fixtures = [ - 'uefi_vm', - 'unix_vm', - 'windows_vm', - 'hostA2', - 'hostB1', - 'sr_disk', - 'sr_disk_4k' - ] - - for item in items: - fixturenames = getattr(item, 'fixturenames', ()) - for fixturename in markable_fixtures: - if fixturename in fixturenames: - item.add_marker(fixturename) - - if 'vm_ref' not in fixturenames: - item.add_marker('no_vm') - - if item.get_closest_marker('multi_vms'): - # multi_vms implies small_vm - item.add_marker('small_vm') diff --git a/data.py-dist b/data.py-dist index a1bc8d3a..ddea97f2 100644 --- a/data.py-dist +++ b/data.py-dist @@ -5,6 +5,12 @@ # You need to have an SSH key into the hosts' /root/.ssh/authorized_keys. HOST_DEFAULT_USER = "root" HOST_DEFAULT_PASSWORD = "" +HOST_DEFAULT_PASSWORD_HASH = "" # FIXME + +# Public key for a private key available to the test runner +TEST_SSH_PUBKEY = """ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMnN/wVdQqHA8KsndfrLS7fktH/IEgxoa533efuXR6rw XCP-ng CI +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDKz9uQOoxq6Q0SQ0XTzQHhDolvuo/7EyrDZsYQbRELhcPJG8MT/o5u3HyJFhIP2+HqBSXXgmqRPJUkwz9wUwb2sUwf44qZm/pyPUWOoxyVtrDXzokU/uiaNKUMhbnfaXMz6Ogovtjua63qld2+ZRXnIgrVtYKtYBeu/qKGVSnf4FTOUKl1w3uKkr59IUwwAO8ay3wVnxXIHI/iJgq6JBgQNHbn3C/SpYU++nqL9G7dMyqGD36QPFuqH/cayL8TjNZ67TgAzsPX8OvmRSqjrv3KFbeSlpS/R4enHkSemhgfc8Z2f49tE7qxWZ6x4Uyp5E6ur37FsRf/tEtKIUJGMRXN XCP-ng CI +""" # The following prefix will be added to the `name-label` parameter of XAPI objects # that the tests will create or import, such as VMs and SRs. @@ -20,18 +26,70 @@ HOSTS = { # "testhost1": {"user": "root", "password": "", 'skip_xo_config': True}, } +NETWORKS = { +# "MGMT": "7a48464c-1e70-4e2e-a449-9f2514564381", +} + # PXE config server for automated XCP-ng installation PXE_CONFIG_SERVER = 'pxe' # Default VM images location DEF_VM_URL = 'http://pxe/images/' +# Default shared ISO SR +ISOSR_SRV = "nfs-server" +ISOSR_PATH = "/srv/iso-sr" + +# IP addresses for hosts to be installed +# NOTE: do NOT set an IP for host1, it is assumed to use DEFAULT +HOSTS_IP_CONFIG = { + 'HOSTS': { +# 'DEFAULT': '192.16.0.1', +# 'host2': '192.16.0.2', + }, +# 'NETMASK': '255.255.0.0', +# 'GATEWAY': '192.16.0.254', +# 'DNS': '192.16.0.254', +} + +# Tools +TOOLS = { +# "iso-remaster": "/home/user/src/xcpng/xcp/scripts/iso-remaster/iso-remaster.sh", +} + # Values can be either full URLs or only partial URLs that will be automatically appended to DEF_VM_URL VM_IMAGES = { 'mini-linux-x86_64-bios': 'alpine-minimal-3.12.0.xva', 'mini-linux-x86_64-uefi': 'alpine-uefi-minimal-3.12.0.xva' } +# FIXME: use URLs and an optional cache? +# +# If 'net-only' is set to 'True' only source of type URL will be possible. +# By default the parameter is set to False. +ISO_IMAGES = { +# '83rc1': {'path': "/home/user/iso/xcp-ng-8.3.0-rc1.iso", +# 'pxe-url': "http://server/installers/xcp-ng/8.3-rc1", +# 'net-url': "http://server/installers/xcp-ng/8.3-rc1"}, +# '83rc1net': {'path': "/home/user/iso/xcp-ng-8.3.0-rc1-netinstall.iso", +# 'net-url': "http://server/installers/xcp-ng/8.3-rc1", +# 'net-only': True}, +# '83b2': {'path': "/home/user/iso/xcp-ng-8.3.0-beta2.iso", +# 'net-url': "http://server/installers/xcp-ng/8.3-beta2"}, +# '821.1': {'path': "/home/user/iso/xcp-ng-8.2.1-20231130.iso"}, +# '821': {'path': "/home/user/iso/xcp-ng-8.2.1.iso"}, +# '820': {'path': "/home/user/iso/xcp-ng-8.2.0.iso"}, +# '81': {'path': "/home/user/iso/xcp-ng-8.1.0-2.iso"}, +# '80': {'path': "/home/user/iso/xcp-ng-8.0.0.iso"}, +# '76': {'path': "/home/user/iso/xcp-ng-7.6.0.iso"}, +# '75': {'path': "/home/user/iso/xcp-ng-7.5.0-2.iso"}, +# +# 'xs8': {'path': "/home/user/iso/XenServer8_2024-03-18.iso"}, +# #'xs8': {'path': "/home/user/iso/XenServer8_2023-08-22.iso"}, +# 'ch821.1': {'path': "/home/user/iso/CitrixHypervisor-8.2.1-2306-install-cd.iso"}, +# 'ch821': {'path': "/home/user/iso/CitrixHypervisor-8.2.1-install-cd.iso"}, +} + # In some cases, we may prefer to favour a local SR to store test VM disks, # to avoid latency or unstabilities related to network or shared file servers. # However it's not good practice to make a local SR the default SR for a pool of several hosts. @@ -98,6 +156,32 @@ LVMOISCSI_DEVICE_CONFIG = { # 'SCSIid': 'id' } +BASE_ANSWERFILES = dict( + INSTALL={ + "TAG": "installation", + "CONTENTS": ( + {"TAG": "root-password", + "type": "hash", + "CONTENTS": HOST_DEFAULT_PASSWORD_HASH}, + {"TAG": "timezone", + "CONTENTS": "Europe/Paris"}, + {"TAG": "keymap", + "CONTENTS": "us"}, + ), + }, + UPGRADE={ + "TAG": "installation", + "mode": "upgrade", + }, + RESTORE={ + "TAG": "restore", + }, +) + +IMAGE_EQUIVS = { +# 'install.test::Nested::install[bios-83rc1-ext]-vm1-607cea0c825a4d578fa5fab56978627d8b2e28bb': 'install.test::Nested::install[bios-83rc1-ext]-vm1-addb4ead4da49856e1d2fb3ddf4e31027c6b693b', +} + # compatibility settings for older tests DEFAULT_NFS_DEVICE_CONFIG = NFS_DEVICE_CONFIG DEFAULT_NFS4_DEVICE_CONFIG = NFS4_DEVICE_CONFIG diff --git a/lib/basevm.py b/lib/basevm.py index 951c6c59..dcd7ae8a 100644 --- a/lib/basevm.py +++ b/lib/basevm.py @@ -2,7 +2,7 @@ import lib.commands as commands -from lib.common import _param_get, _param_remove, _param_set +from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set from lib.sr import SR class BaseVM: @@ -16,13 +16,24 @@ def __init__(self, uuid, host): self.host = host def param_get(self, param_name, key=None, accept_unknown_key=False): - return _param_get(self.host, BaseVM.xe_prefix, self.uuid, param_name, key, accept_unknown_key) + return _param_get(self.host, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) def param_set(self, param_name, value, key=None): - _param_set(self.host, BaseVM.xe_prefix, self.uuid, param_name, value, key) + _param_set(self.host, self.xe_prefix, self.uuid, + param_name, value, key) def param_remove(self, param_name, key, accept_unknown_key=False): - _param_remove(self.host, BaseVM.xe_prefix, self.uuid, param_name, key, accept_unknown_key) + _param_remove(self.host, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) + + def param_add(self, param_name, value, key=None): + _param_add(self.host, self.xe_prefix, self.uuid, + param_name, value, key) + + def param_clear(self, param_name): + _param_clear(self.host, self.xe_prefix, self.uuid, + param_name) def name(self): return self.param_get('name-label') @@ -38,7 +49,7 @@ def vdi_uuids(self, sr_uuid=None): vdis_on_sr = [] for vdi in vdis: - if self.get_vdi_sr_uuid(vdi) == sr_uuid: + if self.host.get_vdi_sr_uuid(vdi) == sr_uuid: vdis_on_sr.append(vdi) return vdis_on_sr @@ -51,19 +62,16 @@ def destroy(self): self.destroy_vdi(vdi_uuid) self._destroy() - def get_vdi_sr_uuid(self, vdi_uuid): - return self.host.xe('vdi-param-get', {'uuid': vdi_uuid, 'param-name': 'sr-uuid'}) - def all_vdis_on_host(self, host): for vdi_uuid in self.vdi_uuids(): - sr = SR(self.get_vdi_sr_uuid(vdi_uuid), self.host.pool) + sr = SR(self.host.get_vdi_sr_uuid(vdi_uuid), self.host.pool) if not sr.attached_to_host(host): return False return True def all_vdis_on_sr(self, sr): for vdi_uuid in self.vdi_uuids(): - if self.get_vdi_sr_uuid(vdi_uuid) != sr.uuid: + if self.host.get_vdi_sr_uuid(vdi_uuid) != sr.uuid: return False return True @@ -71,15 +79,22 @@ def get_sr(self): # in this method we assume the SR of the first VDI is the VM SR vdis = self.vdi_uuids() assert len(vdis) > 0, "Don't ask for the SR of a VM without VDIs!" - sr = SR(self.get_vdi_sr_uuid(vdis[0]), self.host.pool) + sr = SR(self.host.get_vdi_sr_uuid(vdis[0]), self.host.pool) assert sr.attached_to_host(self.host) return sr - def export(self, filepath, compress='none'): - logging.info("Export VM %s to %s with compress=%s" % (self.uuid, filepath, compress)) - params = { - 'uuid': self.uuid, - 'compress': compress, - 'filename': filepath - } - self.host.xe('vm-export', params) + def export(self, filepath, compress='none', use_cache=False): + + if use_cache: + logging.info("Export VM %s to cache for %r as a clone" % (self.uuid, filepath)) + clone = self.clone() + logging.info(f"Marking VM {clone.uuid} as cached") + clone.param_set('name-description', self.host.vm_cache_key(filepath)) + else: + logging.info("Export VM %s to %s with compress=%s" % (self.uuid, filepath, compress)) + params = { + 'uuid': self.uuid, + 'compress': compress, + 'filename': filepath + } + self.host.xe('vm-export', params) diff --git a/lib/commands.py b/lib/commands.py index 841f52e3..78891fbd 100644 --- a/lib/commands.py +++ b/lib/commands.py @@ -26,7 +26,7 @@ def __init__(self, returncode, stdout, cmd): class LocalCommandFailed(BaseCommandFailed): def __init__(self, returncode, stdout, cmd): msg_end = f": {stdout}" if stdout else "." - super(SSHCommandFailed, self).__init__( + super(LocalCommandFailed, self).__init__( returncode, stdout, cmd, f'Local command ({cmd}) failed with return code {returncode}{msg_end}' ) @@ -59,14 +59,8 @@ def _ellide_log_lines(log): reduced_message.append("(...)") return "\n{}".format("\n".join(reduced_message)) -OUPUT_LOGGER = logging.getLogger('output') -OUPUT_LOGGER.propagate = False -OUTPUT_HANDLER = logging.StreamHandler() -OUPUT_LOGGER.addHandler(OUTPUT_HANDLER) -OUTPUT_HANDLER.setFormatter(logging.Formatter('%(message)s')) - -def _ssh(hostname_or_ip, cmd, check=True, simple_output=True, suppress_fingerprint_warnings=True, - background=False, target_os='linux', decode=True, options=[]): +def _ssh(hostname_or_ip, cmd, check, simple_output, suppress_fingerprint_warnings, + background, target_os, decode, options): opts = list(options) opts.append('-o "BatchMode yes"') if suppress_fingerprint_warnings: @@ -111,7 +105,7 @@ def _ssh(hostname_or_ip, cmd, check=True, simple_output=True, suppress_fingerpri for line in iter(process.stdout.readline, b''): readable_line = line.decode(errors='replace').strip() stdout.append(line) - OUPUT_LOGGER.debug(readable_line) + logging.debug("> %s", readable_line) _, stderr = process.communicate() res = subprocess.CompletedProcess(ssh_cmd, process.returncode, b''.join(stdout), stderr) @@ -205,6 +199,7 @@ def sftp(hostname_or_ip, cmds, check=True, suppress_fingerprint_warnings=True): def local_cmd(cmd, check=True, decode=True): """ Run a command locally on tester end. """ + logging.debug("[local] %s", (cmd,)) res = subprocess.run( cmd, stdout=subprocess.PIPE, diff --git a/lib/common.py b/lib/common.py index 66c8e4b0..52d6a893 100644 --- a/lib/common.py +++ b/lib/common.py @@ -1,11 +1,15 @@ import getpass import inspect +import itertools import logging +import sys import time import traceback from enum import Enum from uuid import UUID +import pytest + import lib.commands as commands class PackageManagerEnum(Enum): @@ -29,6 +33,51 @@ def prefix_object_name(label): name_prefix = f"[{getpass.getuser()}]" return f"{name_prefix} {label}" +def shortened_nodeid(nodeid): + components = nodeid.split("::") + # module + components[0] = strip_prefix(components[0], "tests/") + components[0] = strip_suffix(components[0], ".py") + components[0] = components[0].replace("/", ".") + # function + components[-1] = strip_prefix(components[-1], "test_") + # class + if len(components) > 2: + components[1] = strip_prefix(components[1], "Test") + + return "::".join(components) + +def expand_scope_relative_nodeid(scoped_nodeid, scope, ref_nodeid): + if scope == 'session' or scope == 'package': + base = () + elif scope == 'module': + base = ref_nodeid.split("::", 1)[:1] + elif scope == 'class': + base = ref_nodeid.split("::", 2)[:2] + else: + raise RuntimeError(f"Internal error: invalid scope {scope!r}") + logging.debug("scope: %r base: %r relative: %r", scope, base, scoped_nodeid) + return "::".join(itertools.chain(base, (scoped_nodeid,))) + +def callable_marker(value, request, param_mapping): + """Process value optionally generated by fixture-dependent callable. + + Typically useful for fixtures using pytest markers on parametrized tests. + + Such markers take 2 parameters: + - one or more value(s), or callable(s) that will return a value + - a mapping of parameters to test fixture names, to pass as named arguments + to the callable + """ + if callable(value): + try: + params = { arg_name: request.getfixturevalue(fixture_name) + for arg_name, fixture_name in param_mapping.items() } + except pytest.FixtureLookupError as e: + raise RuntimeError("fixture in mapping not found on test") from e + value = value(**params) + return value + def wait_for(fn, msg=None, timeout_secs=2 * 60, retry_delay_secs=2, invert=False): if msg is not None: logging.info(msg) @@ -75,6 +124,20 @@ def safe_split(text, sep=','): """ A split function that returns an empty list if the input string is empty. """ return text.split(sep) if len(text) > 0 else [] +def strip_prefix(string, prefix): + if sys.version_info >= (3, 9): + return string.removeprefix(prefix) + if string.startswith(prefix): + return string[len(prefix):] + return string + +def strip_suffix(string, suffix): + if sys.version_info >= (3, 9): + return string.removesuffix(suffix) + if string.endswith(suffix): + return string[:-len(suffix)] + return string + def setup_formatted_and_mounted_disk(host, sr_disk, fs_type, mountpoint): if fs_type == 'ext4': option_force = '-F' diff --git a/lib/host.py b/lib/host.py index 712c2053..86b1023a 100644 --- a/lib/host.py +++ b/lib/host.py @@ -8,7 +8,8 @@ import lib.commands as commands -from lib.common import _param_get, safe_split, to_xapi_bool, wait_for, wait_for_not +from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set +from lib.common import safe_split, strip_suffix, to_xapi_bool, wait_for, wait_for_not from lib.common import prefix_object_name from lib.netutil import wrap_ip from lib.sr import SR @@ -31,6 +32,7 @@ class Host: xe_prefix = "host" def __init__(self, pool, hostname_or_ip): + self.nested = None # if running nested, the VM object for this host self.pool = pool self.hostname_or_ip = hostname_or_ip self.inventory = None @@ -97,7 +99,24 @@ def stringify(key, value): return result def param_get(self, param_name, key=None, accept_unknown_key=False): - return _param_get(self, Host.xe_prefix, self.uuid, param_name, key, accept_unknown_key) + return _param_get(self, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) + + def param_set(self, param_name, value, key=None): + _param_set(self, self.xe_prefix, self.uuid, + param_name, value, key) + + def param_remove(self, param_name, key, accept_unknown_key=False): + _param_remove(self, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) + + def param_add(self, param_name, value, key=None): + _param_add(self, self.xe_prefix, self.uuid, + param_name, value, key) + + def param_clear(self, param_name): + _param_clear(self, self.xe_prefix, self.uuid, + param_name) def create_file(self, filename, text): with tempfile.NamedTemporaryFile('w') as file: @@ -200,21 +219,46 @@ def xo_server_reconnect(self): # is not enough to guarantee that the host object exists yet. wait_for(lambda: xo_object_exists(self.uuid), "Wait for XO to know about HOST %s" % self.uuid) + @staticmethod + def vm_cache_key(uri): + return f"[Cache for {strip_suffix(uri, '.xva')}]" + + def cached_vm(self, uri, sr_uuid): + assert sr_uuid, "A SR UUID is necessary to use import cache" + cache_key = self.vm_cache_key(uri) + # Look for an existing cache VM + vm_uuids = safe_split(self.xe('vm-list', {'name-description': cache_key}, minimal=True), ',') + + for vm_uuid in vm_uuids: + vm = VM(vm_uuid, self) + # Make sure the VM is on the wanted SR. + # Assumption: if the first disk is on the SR, the VM is. + # If there's no VDI at all, then it is virtually on any SR. + if not vm.vdi_uuids() or vm.get_sr().uuid == sr_uuid: + logging.info(f"Reusing cached VM {vm.uuid} for {uri}") + return vm + logging.info("Could not find a VM in cache with key %r", cache_key) + def import_vm(self, uri, sr_uuid=None, use_cache=False): + vm = None if use_cache: - assert sr_uuid, "A SR UUID is necessary to use import cache" - cache_key = f"[Cache for {uri}]" - # Look for an existing cache VM - vm_uuids = safe_split(self.xe('vm-list', {'name-description': cache_key}, minimal=True), ',') - - for vm_uuid in vm_uuids: - vm = VM(vm_uuid, self) - # Make sure the VM is on the wanted SR. - # Assumption: if the first disk is on the SR, the VM is. - # If there's no VDI at all, then it is virtually on any SR. - if not vm.vdi_uuids() or vm.get_sr().uuid == sr_uuid: - logging.info(f"Reusing cached VM {vm.uuid} for {uri}") - return vm + if '://' in uri and uri.startswith("clone"): + protocol, rest = uri.split(":", 1) + assert rest.startswith("//") + filename = rest[2:] # strip "//" + base_vm = self.cached_vm(filename, sr_uuid) + if base_vm: + vm = base_vm.clone() + vm.param_clear('name-description') + if uri.startswith("clone+start"): + vm.start() + wait_for(vm.is_running, "Wait for VM running") + else: + vm = self.cached_vm(uri, sr_uuid) + if vm: + return vm + else: + assert not ('://' in uri and uri.startswith("clone")), "clone URIs require cache enabled" params = {} msg = "Import VM %s" % uri @@ -227,7 +271,6 @@ def import_vm(self, uri, sr_uuid=None, use_cache=False): params['sr-uuid'] = sr_uuid logging.info(msg) vm_uuid = self.xe('vm-import', params) - logging.info("VM UUID: %s" % vm_uuid) vm_name = prefix_object_name(self.xe('vm-param-get', {'uuid': vm_uuid, 'param-name': 'name-label'})) vm = VM(vm_uuid, self) vm.param_set('name-label', vm_name) @@ -235,10 +278,20 @@ def import_vm(self, uri, sr_uuid=None, use_cache=False): for vif in vm.vifs(): vif.move(self.management_network()) if use_cache: + cache_key = self.vm_cache_key(uri) logging.info(f"Marking VM {vm.uuid} as cached") vm.param_set('name-description', cache_key) return vm + def vm_from_template(self, name, template): + params = { + "new-name-label": prefix_object_name(name), + "template": template, + "sr-uuid": self.main_sr_uuid(), + } + vm_uuid = self.xe('vm-install', params) + return VM(vm_uuid, self) + def pool_has_vm(self, vm_uuid, vm_type='vm'): if vm_type == 'snapshot': return self.xe('snapshot-list', {'uuid': vm_uuid}, minimal=True) == vm_uuid @@ -373,6 +426,17 @@ def yum_restore_saved_state(self): self.saved_packages_list = None self.saved_rollback_id = None + def shutdown(self, verify=False): + logging.info("Shutdown host %s" % self) + try: + self.ssh(['shutdown']) + except commands.SSHCommandFailed as e: + # ssh connection may get killed by the shutdown and terminate with an error code + if "closed by remote host" not in e.stdout: + raise + if verify: + wait_for_not(self.is_enabled, "Wait for host down") + def reboot(self, verify=False): logging.info("Reboot host %s" % self) try: @@ -460,7 +524,7 @@ def local_vm_srs(self): srs.append(sr) return srs - def main_sr(self): + def main_sr_uuid(self): """ Main SR is either the default SR, or the first local SR, depending on data.py's DEFAULT_SR. """ try: from data import DEFAULT_SR @@ -470,9 +534,11 @@ def main_sr(self): sr_uuid = None if DEFAULT_SR == 'local': + hostname = self.xe('host-param-get', {'uuid': self.uuid, + 'param-name': 'name-label'}) local_sr_uuids = safe_split( # xe sr-list doesn't support filtering by host UUID! - self.ssh(['xe sr-list host=$HOSTNAME content-type=user minimal=true']), + self.xe('sr-list', {'host': hostname, 'content-type': 'user', 'minimal': 'true'}), ',' ) assert local_sr_uuids, f"DEFAULT_SR=='local' so there must be a local SR on host {self}" @@ -480,6 +546,7 @@ def main_sr(self): else: sr_uuid = self.pool.param_get('default-SR') assert sr_uuid, f"DEFAULT_SR='default' so there must be a default SR on the pool of host {self}" + assert sr_uuid != "" return sr_uuid def hostname(self): @@ -509,6 +576,7 @@ def join_pool(self, pool): lambda: master.xe('host-param-get', {'uuid': self.uuid, 'param-name': 'enabled'}), f"Wait for pool {master} to see joined host {self} as enabled." ) + self.pool = pool def activate_smapi_driver(self, driver): sm_plugins = self.ssh(['grep', '[[:space:]]*sm-plugins[[:space:]]*=[[:space:]]*', XAPI_CONF_FILE]).splitlines() @@ -534,3 +602,6 @@ def enable_hsts_header(self): def disable_hsts_header(self): self.ssh(['rm', '-f', f'{XAPI_CONF_DIR}/00-XCP-ng-tests-enable-hsts-header.conf']) self.restart_toolstack(verify=True) + + def get_vdi_sr_uuid(self, vdi_uuid): + return self.xe('vdi-param-get', {'uuid': vdi_uuid, 'param-name': 'sr-uuid'}) diff --git a/lib/installer.py b/lib/installer.py new file mode 100644 index 00000000..1f0ac398 --- /dev/null +++ b/lib/installer.py @@ -0,0 +1,283 @@ +import atexit +import logging +import tempfile +import time +import xml.etree.ElementTree as ET + +from lib import commands, pxe +from lib.commands import local_cmd, ssh +from lib.common import wait_for + +from data import HOST_DEFAULT_PASSWORD_HASH +from data import ISO_IMAGES + +def clean_files_on_pxe(mac_address): + logging.info('cleanning file for mac {}'.format(mac_address)) + pxe.server_remove_config(mac_address) + +def setup_pxe_boot(vm, mac_address, version): + with tempfile.TemporaryDirectory(suffix=mac_address) as tmp_local_path: + logging.info('Generate answerfile.xml for {}'.format(mac_address)) + + hdd = 'nvme0n1' if vm.is_uefi else 'sda' + encrypted_password = HOST_DEFAULT_PASSWORD_HASH + try: + installer = ISO_IMAGES[version]['pxe-url'] + except KeyError: + pytest.skip(f"cannot found pxe-url for {version}") + pxe_addr = pxe.PXE_CONFIG_SERVER + + with open(f'{tmp_local_path}/answerfile.xml', 'w') as answerfile: + answerfile.write(f""" + + fr + {hdd} + {hdd} + {encrypted_password} + {installer} + + Europe/Paris + + + """) + + logging.info('Generate boot.conf for {}'.format(mac_address)) + pxe.generate_boot_conf(tmp_local_path, installer) + + logging.info('Copy files to {}'.format(pxe_addr)) + pxe.server_push_config(mac_address, tmp_local_path) + + logging.info('Register cleanup files for PXE') + atexit.register(lambda: clean_files_on_pxe(mac_address)) + +class AnswerFile: + def __init__(self, kind, /): + from data import BASE_ANSWERFILES + defn = BASE_ANSWERFILES[kind] + self.defn = self._normalize_structure(defn) + + def write_xml(self, filename): + logging.info("generating answerfile %s", filename) + etree = ET.ElementTree(self._defn_to_xml_et(self.defn)) + etree.write(filename) + + # chainable mutators for lambdas + + def top_append(self, *defs): + for defn in defs: + self.defn['CONTENTS'].append(self._normalize_structure(defn)) + return self + + def top_setattr(self, attrs): + assert 'CONTENTS' not in attrs + self.defn.update(attrs) + return self + + # makes a mutable deep copy of all `contents` + @staticmethod + def _normalize_structure(defn): + assert isinstance(defn, dict) + assert 'TAG' in defn + defn = dict(defn) + if 'CONTENTS' not in defn: + defn['CONTENTS'] = [] + if not isinstance(defn['CONTENTS'], str): + defn['CONTENTS'] = [AnswerFile._normalize_structure(item) + for item in defn['CONTENTS']] + return defn + + # convert to a ElementTree.Element tree suitable for further + # modification before we serialize it to XML + @staticmethod + def _defn_to_xml_et(defn, /, *, parent=None): + assert isinstance(defn, dict) + defn = dict(defn) + name = defn.pop('TAG') + assert isinstance(name, str) + contents = defn.pop('CONTENTS', ()) + assert isinstance(contents, (str, list)) + element = ET.Element(name, **defn) + if parent is not None: + parent.append(element) + if isinstance(contents, str): + element.text = contents + else: + for contents in contents: + AnswerFile._defn_to_xml_et(contents, parent=element) + return element + +def poweroff(ip): + try: + ssh(ip, ["poweroff"]) + except commands.SSHCommandFailed as e: + # ignore connection closed by reboot + if e.returncode == 255 and "closed by remote host" in e.stdout: + logging.info("sshd closed the connection") + pass + else: + raise + +def monitor_install(*, ip, is_pxe=False): + if is_pxe: + # When using PXE boot the IP is available but ssh is not so we just + # need to wait. + while True: + try: + ssh(ip, ["ls"], check=False) + except commands.SSHCommandFailed as e: + if e.returncode == 255: + logging.debug("ssh connexion refused, wait 10s...") + time.sleep(10) + else: + # Raise all other errors + raise e + else: + logging.debug("ssh works") + break + else: + # wait for "yum install" phase to finish + wait_for(lambda: ssh(ip, ["grep", + "'DISPATCH: NEW PHASE: Completing installation'", + "/tmp/install-log"], + check=False, simple_output=False, + ).returncode == 0, + "Wait for rpm installation to succeed", + timeout_secs=40*60) # FIXME too big + + # wait for install to finish + wait_for(lambda: ssh(ip, ["grep", + "'The installation completed successfully'", + "/tmp/install-log"], + check=False, simple_output=False, + ).returncode == 0, + "Wait for system installation to succeed", + timeout_secs=40*60) # FIXME too big + + wait_for(lambda: ssh(ip, ["ps a|grep '[0-9]. python /opt/xensource/installer/init'"], + check=False, simple_output=False, + ).returncode == 1, + "Wait for installer to terminate") + +def perform_install(*, iso, host_vm, version=None): + vif = host_vm.vifs()[0] + mac_address = vif.param_get('MAC') + logging.info("Host VM has MAC %s", mac_address) + + if iso: + host_vm.insert_cd(iso) + else: + setup_pxe_boot(host_vm, mac_address, version) + + try: + host_vm.start() + wait_for(host_vm.is_running, "Wait for host VM running") + + # catch host-vm IP address + wait_for(lambda: pxe.arp_addresses_for(mac_address), + "Wait for DHCP server to see Host VM in ARP tables", + timeout_secs=10*60) + ips = pxe.arp_addresses_for(mac_address) + logging.info("Host VM has IPs %s", ips) + assert len(ips) == 1 + host_vm.ip = ips[0] + + monitor_install(ip=host_vm.ip, is_pxe = False if iso else True) + + logging.info("Shutting down Host VM after successful installation") + poweroff(host_vm.ip) + wait_for(host_vm.is_halted, "Wait for host VM halted") + if iso: + host_vm.eject_cd() + + except Exception as e: + logging.critical("caught exception %s", e) + # wait_for(lambda: False, 'Wait "forever"', timeout_secs=100*60) + #host_vm.shutdown(force=True) + raise + except KeyboardInterrupt: + logging.warning("keyboard interrupt") + # wait_for(lambda: False, 'Wait "forever"', timeout_secs=100*60) + #host_vm.shutdown(force=True) + raise + +def monitor_upgrade(*, ip): + # wait for "yum install" phase to start + wait_for(lambda: ssh(ip, ["grep", + "'DISPATCH: NEW PHASE: Reading package information'", + "/tmp/install-log"], + check=False, simple_output=False, + ).returncode == 0, + "Wait for upgrade preparations to finish", + timeout_secs=40*60) # FIXME too big + + # wait for "yum install" phase to finish + wait_for(lambda: ssh(ip, ["grep", + "'DISPATCH: NEW PHASE: Completing installation'", + "/tmp/install-log"], + check=False, simple_output=False, + ).returncode == 0, + "Wait for rpm installation to succeed", + timeout_secs=40*60) # FIXME too big + + # wait for install to finish + wait_for(lambda: ssh(ip, ["grep", + "'The installation completed successfully'", + "/tmp/install-log"], + check=False, simple_output=False, + ).returncode == 0, + "Wait for system installation to succeed", + timeout_secs=40*60) # FIXME too big + + wait_for(lambda: ssh(ip, ["ps a|grep '[0-9]. python /opt/xensource/installer/init'"], + check=False, simple_output=False, + ).returncode == 1, + "Wait for installer to terminate") + +def perform_upgrade(*, iso, host_vm): + vif = host_vm.vifs()[0] + mac_address = vif.param_get('MAC') + logging.info("Host VM has MAC %s", mac_address) + + host_vm.insert_cd(iso) + + try: + pxe.arp_clear_for(mac_address) + + host_vm.start() + wait_for(host_vm.is_running, "Wait for host VM running") + + # catch host-vm IP address + wait_for(lambda: pxe.arp_addresses_for(mac_address), + "Wait for DHCP server to see Host VM in ARP tables", + timeout_secs=10*60) + ips = pxe.arp_addresses_for(mac_address) + logging.info("Host VM has IPs %s", ips) + assert len(ips) == 1 + host_vm.ip = ips[0] + + # host may not be up if ARP cache was filled + wait_for(lambda: local_cmd(["ping", "-c1", host_vm.ip], check=False), + "Wait for host up", timeout_secs=10 * 60, retry_delay_secs=10) + wait_for(lambda: local_cmd(["nc", "-zw5", host_vm.ip, "22"], check=False), + "Wait for ssh up on host", timeout_secs=10 * 60, retry_delay_secs=5) + + monitor_upgrade(ip=host_vm.ip) + + logging.info("Shutting down Host VM after successful upgrade") + poweroff(host_vm.ip) + wait_for(host_vm.is_halted, "Wait for host VM halted") + + except Exception as e: + logging.critical("caught exception %s", e) + # wait_for(lambda: False, 'Wait "forever"', timeout_secs=100*60) + #host_vm.shutdown(force=True) + raise + except KeyboardInterrupt: + logging.warning("keyboard interrupt") + # wait_for(lambda: False, 'Wait "forever"', timeout_secs=100*60) + #host_vm.shutdown(force=True) + raise + + host_vm.eject_cd() diff --git a/lib/pif.py b/lib/pif.py new file mode 100644 index 00000000..5991f391 --- /dev/null +++ b/lib/pif.py @@ -0,0 +1,30 @@ +import lib.commands as commands + +from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set + +class PIF: + xe_prefix = "pif" + + def __init__(self, uuid, host): + self.uuid = uuid + self.host = host + + def param_get(self, param_name, key=None, accept_unknown_key=False): + return _param_get(self.host, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) + + def param_set(self, param_name, value, key=None): + _param_set(self.host, self.xe_prefix, self.uuid, + param_name, value, key) + + def param_remove(self, param_name, key, accept_unknown_key=False): + _param_remove(self.host, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) + + def param_add(self, param_name, value, key=None): + _param_add(self.host, self.xe_prefix, self.uuid, + param_name, value, key) + + def param_clear(self, param_name): + _param_clear(self.host, self.xe_prefix, self.uuid, + param_name) diff --git a/lib/pxe.py b/lib/pxe.py new file mode 100644 index 00000000..7a720521 --- /dev/null +++ b/lib/pxe.py @@ -0,0 +1,57 @@ +from lib.commands import ssh, scp, SSHCommandFailed + +PXE_CONFIG_DIR = "/pxe/configs/custom" + +try: + from data import PXE_CONFIG_SERVER + assert PXE_CONFIG_SERVER +except ImportError: + raise Exception('No address for the PXE server found in data.py (`PXE_CONFIG_SERVER`)') + +def generate_boot_conf(directory, installer): + if True: + rt = "" + else: + # (because of the restore case), we disable the text ui from + # the installer completely, to workaround a bug that leaves us + # stuck on a confirmation dialog at the end of the operation. + rt = 'rt=1' + with open(f'{directory}/boot.conf', 'w') as bootfile: + bootfile.write(f""" +answerfile=custom +installer={installer} +is_default=1 +{rt} +""") + +def server_push_config(mac_address, tmp_local_path): + remote_dir = f'{PXE_CONFIG_DIR}/{mac_address}/' + server_remove_config(mac_address) + ssh(PXE_CONFIG_SERVER, ['mkdir', '-p', remote_dir]) + scp(PXE_CONFIG_SERVER, f'{tmp_local_path}/boot.conf', remote_dir) + scp(PXE_CONFIG_SERVER, f'{tmp_local_path}/answerfile.xml', remote_dir) + +def server_remove_config(mac_address): + assert mac_address # protection against deleting the whole parent dir! + remote_dir = f'{PXE_CONFIG_DIR}/{mac_address}/' + ssh(PXE_CONFIG_SERVER, ['rm', '-rf', remote_dir]) + +def server_remove_bootconf(mac_address): + assert mac_address + distant_file = f'{PXE_CONFIG_DIR}/{mac_address}/boot.conf' + ssh(PXE_CONFIG_SERVER, ['rm', '-rf', distant_file]) + +def arp_addresses_for(mac_address): + output = ssh( + PXE_CONFIG_SERVER, + ['arp', '-n', '|', 'grep', mac_address, '|', 'awk', '\'{ print $1 }\''] + ) + candidate_ips = output.splitlines() + return candidate_ips + +def arp_clear_for(mac_address): + for stray_ip in arp_addresses_for(mac_address): + output = ssh( + PXE_CONFIG_SERVER, + ['arp', '-d', stray_ip] + ) diff --git a/lib/sr.py b/lib/sr.py index c36462ba..7dd8e142 100644 --- a/lib/sr.py +++ b/lib/sr.py @@ -157,7 +157,7 @@ def create_vdi(self, name_label, virtual_size=64): 'virtual-size': str(virtual_size), 'sr-uuid': self.uuid }) - return VDI(self, vdi_uuid) + return VDI(vdi_uuid, sr=self) def run_quicktest(self): logging.info(f"Run quicktest on SR {self.uuid}") diff --git a/lib/vbd.py b/lib/vbd.py new file mode 100644 index 00000000..fb535cac --- /dev/null +++ b/lib/vbd.py @@ -0,0 +1,44 @@ +import logging + +from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set + +class VBD: + xe_prefix = "vbd" + + def __init__(self, uuid, vm, device): + self.uuid = uuid + self.vm = vm + self.device = device + + def plug(self): + self.vm.host.xe("vbd-plug", {'uuid': self.uuid}) + + def unplug(self): + self.vm.host.xe("vbd-unplug", {'uuid': self.uuid}) + + def param_get(self, param_name, key=None, accept_unknown_key=False): + return _param_get(self.vm.host, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) + + def param_set(self, param_name, value, key=None): + _param_set(self.vm.host, self.xe_prefix, self.uuid, + param_name, value, key) + + def param_remove(self, param_name, key, accept_unknown_key=False): + _param_remove(self.vm.host, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) + + def param_add(self, param_name, value, key=None): + _param_add(self.vm.host, self.xe_prefix, self.uuid, + param_name, value, key) + + def param_clear(self, param_name): + _param_clear(self.vm.host, self.xe_prefix, self.uuid, + param_name) + + def destroy(self): + logging.info("Destroy %s", self) + self.vm.host.pool.master.xe('vbd-destroy', {'uuid': self.uuid}) + + def __str__(self): + return f"VBD {self.uuid} for {self.device} of VM {self.vm.uuid}" diff --git a/lib/vdi.py b/lib/vdi.py index e9f82b18..3b331731 100644 --- a/lib/vdi.py +++ b/lib/vdi.py @@ -1,14 +1,52 @@ import logging +from lib.common import _param_add, _param_clear, _param_get, _param_remove, _param_set + class VDI: - def __init__(self, sr, uuid): + xe_prefix = "vdi" + + def __init__(self, uuid, *, host=None, sr=None): self.uuid = uuid # TODO: use a different approach when migration is possible - self.sr = sr + if sr is None: + sr_uuid = host.get_vdi_sr_uuid(uuid) + # avoid circular import + # FIXME should get it from Host instead + from lib.sr import SR + self.sr = SR(sr_uuid, host.pool) + else: + self.sr = sr def destroy(self): logging.info("Destroy %s", self) self.sr.pool.master.xe('vdi-destroy', {'uuid': self.uuid}) + def clone(self): + uuid = self.sr.pool.master.xe('vdi-clone', {'uuid': self.uuid}) + return VDI(uuid, sr=self.sr) + + def readonly(self): + return self.param_get("read-only") == "true" + def __str__(self): return f"VDI {self.uuid} on SR {self.sr.uuid}" + + def param_get(self, param_name, key=None, accept_unknown_key=False): + return _param_get(self.sr.pool.master, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) + + def param_set(self, param_name, value, key=None): + _param_set(self.sr.pool.master, self.xe_prefix, self.uuid, + param_name, value, key) + + def param_add(self, param_name, value, key=None): + _param_add(self.sr.pool.master, self.xe_prefix, self.uuid, + param_name, value, key) + + def param_clear(self, param_name): + _param_clear(self.sr.pool.master, self.xe_prefix, self.uuid, + param_name) + + def param_remove(self, param_name, key, accept_unknown_key=False): + _param_remove(self.sr.pool.master, self.xe_prefix, self.uuid, + param_name, key, accept_unknown_key) diff --git a/lib/vm.py b/lib/vm.py index f21c0d5c..bcf8d9ae 100644 --- a/lib/vm.py +++ b/lib/vm.py @@ -8,7 +8,9 @@ from lib.basevm import BaseVM from lib.common import PackageManagerEnum, parse_xe_dict, safe_split, wait_for, wait_for_not +from lib.common import shortened_nodeid, expand_scope_relative_nodeid from lib.snapshot import Snapshot +from lib.vbd import VBD from lib.vif import VIF class VM(BaseVM): @@ -122,7 +124,7 @@ def wait_for_os_booted(self): # waiting for the IP: # - allows to make sure the OS actually started (on VMs that have the management agent) # - allows to store the IP for future use in the VM object - wait_for(self.try_get_and_store_ip, "Wait for VM IP") + wait_for(self.try_get_and_store_ip, "Wait for VM IP", timeout_secs=5 * 60) # now wait also for the management agent to have started wait_for(self.is_management_agent_up, "Wait for management agent up") @@ -247,6 +249,14 @@ def vifs(self): _vifs.append(VIF(vif_uuid, self)) return _vifs + # FIXME: use network_name instead? + def create_vif(self, vif_num, network_uuid): + logging.info("Create VIF %d to network %r on VM %s", vif_num, network_uuid, self.uuid) + self.host.xe('vif-create', {'vm-uuid': self.uuid, + 'device': str(vif_num), + 'network-uuid': network_uuid, + }) + def is_running_on_host(self, host): return self.is_running() and self.param_get('resident-on') == host.uuid @@ -322,7 +332,7 @@ def tools_version(self): return "{major}.{minor}.{micro}-{build}".format(**version_dict) def file_exists(self, filepath): - """ Test that the file at filepath exists. """ + """Returns True if the file exists, otherwise returns False.""" return self.ssh_with_result(['test', '-f', filepath]).returncode == 0 def detect_package_manager(self): @@ -334,10 +344,23 @@ def detect_package_manager(self): else: return PackageManagerEnum.UNKNOWN - def mount_guest_tools_iso(self): - self.host.xe('vm-cd-insert', {'uuid': self.uuid, 'cd-name': 'guest-tools.iso'}) - - def unmount_guest_tools_iso(self): + def insert_cd(self, vdi_name, use_cache=False): + logging.info("Insert CD %r in VM %s", vdi_name, self.uuid) + # FIXME: rescan only the right ISO (if at all possible) + # FIXME: rescan after copying rather than on each use (this + # function can be used also for standard ISOs not generated by our tests) + if not use_cache: + sr_iso_uuids = self.host.xe('sr-list', {'type': 'iso'}, minimal=True).split(',') + for uuid in sr_iso_uuids: + logging.info("scanning SR %s", uuid) + self.host.xe('sr-scan', {'uuid': uuid}) + self.host.xe('vm-cd-insert', {'uuid': self.uuid, 'cd-name': vdi_name}) + + def insert_guest_tools_iso(self): + self.insert_cd('guest-tools.iso') + + def eject_cd(self): + logging.info("Ejecting CD from VM %s", self.uuid) self.host.xe('vm-cd-eject', {'uuid': self.uuid}) # *** Common reusable test fragments @@ -429,10 +452,6 @@ def clear_uefi_variables(self): """ self.param_remove('NVRAM', 'EFI-variables') - def file_exists(self, filepath): - """Returns True if the file exists, otherwise returns False.""" - return self.ssh_with_result(['test', '-f', filepath]).returncode == 0 - def sign_bins(self): for f in self.get_all_efi_bins(): self.sign(f) @@ -472,11 +491,33 @@ def destroy_vtpm(self): logging.info("Destroying vTPM %s" % vtpm_uuid) return self.host.xe('vtpm-destroy', {'uuid': vtpm_uuid}, force=True) + def create_vbd(self, device, vdi_uuid): + logging.info("Create VBD %r for VDI %r on VM %s", device, vdi_uuid, self.uuid) + vbd_uuid = self.host.xe('vbd-create', {'vm-uuid': self.uuid, + 'device': device, + 'vdi-uuid': vdi_uuid, + }) + logging.info("New VBD %s", vbd_uuid) + return VBD(vbd_uuid, self, device) + + def create_cd_vbd(self, device, userdevice): + logging.info("Create CD VBD %r on VM %s", device, self.uuid) + vbd_uuid = self.host.xe('vbd-create', {'vm-uuid': self.uuid, + 'device': device, + 'type': 'CD', + 'mode': 'RO', + }) + vbd = VBD(vbd_uuid, self, device) + vbd.param_set(param_name="userdevice", value=userdevice) + logging.info("New VBD %s", vbd_uuid) + return vbd + def clone(self): - name = self.name() + '_clone_for_tests' + name = self.name() + if not name.endswith('_clone_for_tests'): + name += '_clone_for_tests' logging.info("Clone VM") uuid = self.host.xe('vm-clone', {'uuid': self.uuid, 'new-name-label': name}) - logging.info("New VM: %s (%s)" % (uuid, name)) return VM(uuid, self.host) def install_uefi_certs(self, auths): @@ -586,3 +627,19 @@ def is_cert_present(vm, key): res = vm.host.ssh(['varstore-get', vm.uuid, efi.get_secure_boot_guid(key).as_str(), key], check=False, simple_output=False, decode=False) return res.returncode == 0 + + +def xva_name_from_def(vm_def, ref_nodeid, test_gitref): + vm_name = vm_def["name"] + image_test = vm_def["image_test"] + image_vm = vm_def.get("image_vm", vm_name) + image_scope = vm_def.get("image_scope", "module") + + image_key = "{}-{}-{}".format( + shortened_nodeid(expand_scope_relative_nodeid(image_test, image_scope, ref_nodeid)), + image_vm, + test_gitref) + + from data import IMAGE_EQUIVS + image_key = IMAGE_EQUIVS.get(image_key, image_key) + return f"{image_key}.xva" diff --git a/pytest.ini b/pytest.ini index 4aae1f6d..4cae98f3 100644 --- a/pytest.ini +++ b/pytest.ini @@ -3,7 +3,6 @@ addopts = -ra --maxfail=1 -s markers = # *** Markers that change test behaviour *** default_vm: mark a test with a default VM in case no --vm parameter was given. - incremental: mark a class so that any test failure skips the rest of the tests in the class. # *** Markers used to select tests at collect stage *** @@ -19,6 +18,14 @@ markers = unix_vm: tests that require a unix/linux VM to run. windows_vm: tests that require a Windows VM to run. + # * VM-related markers to give parameters to fixtures + vm_definitions: dicts of VM nick to VM defs for create_vms fixture. + continuation_of: dicts of VM nick to test (and soon VM nick) from which to start + + # * installation-related markers to customize installer run + answerfile: dict defining an answerfile + installer_iso: key of data.ISO_IMAGES identifying an installer ISO + # * Test targets related to VMs small_vm: tests that it is enough to run just once, using the smallest possible VM. big_vm: tests that it would be good to run with a big VM. @@ -29,9 +36,10 @@ markers = flaky: flaky tests. Usually pass, but sometimes fail unexpectedly. complex_prerequisites: tests whose prerequisites are complex and may require special attention. quicktest: runs `quicktest`. + source_type: generate the source type. log_cli = 1 log_cli_level = info -log_cli_format = %(asctime)s %(levelname)s %(message)s -log_cli_date_format = %b %d %H:%M:%S +log_format = %(asctime)s.%(msecs)03d %(levelname)s %(message)s +log_date_format = %b %d %H:%M:%S filterwarnings = error xfail_strict=true diff --git a/requirements/base.txt b/requirements/base.txt index 34fb262e..5d218bec 100644 --- a/requirements/base.txt +++ b/requirements/base.txt @@ -1,14 +1,6 @@ -attrs>=20.3.0 -cffi>=1.14.4 cryptography>=3.3.1 -importlib-metadata>=2.1.1 -more-itertools>=8.6.0 +GitPython packaging>=20.7 -pathlib2>=2.3.5 -pluggy>=0.13.1 -py>=1.9.0 -pyparsing>=2.4.7 -pytest>=5.4.0 -six>=1.15.0 -wcwidth>=0.2.5 -zipp>=1.2.0 +pytest>=8.0.0 +pluggy>=1.1.0 +pytest-dependency diff --git a/requirements/dev.txt b/requirements/dev.txt index 0853008d..3b8d7bb1 100644 --- a/requirements/dev.txt +++ b/requirements/dev.txt @@ -1,13 +1,6 @@ # All requirements + those only used in development ansible>=5.0.1 -ansible-core>=2.12.1 -beautifulsoup4>=4.10.0 bs4>=0.0.1 -Jinja2>=3.0.3 -MarkupSafe>=2.0.1 pycodestyle>=2.6.0 -pycparser>=2.21 PyYAML>=6.0 -resolvelib>=0.5.4 -soupsieve>=2.3.1 -r base.txt diff --git a/scripts/install_xcpng.py b/scripts/install_xcpng.py index 47a716f2..7af91437 100755 --- a/scripts/install_xcpng.py +++ b/scripts/install_xcpng.py @@ -15,6 +15,7 @@ from packaging import version sys.path.append(f"{os.path.abspath(os.path.dirname(__file__))}/..") # noqa +from lib import pxe from lib.commands import ssh, scp, SSHCommandFailed from lib.common import wait_for, is_uuid from lib.host import host_data @@ -23,25 +24,8 @@ logging.basicConfig(format='[%(levelname)s] %(message)s', level=logging.INFO) -PXE_CONFIG_DIR = "/pxe/configs/custom" - def pxe_address(): - try: - from data import PXE_CONFIG_SERVER - return PXE_CONFIG_SERVER - except ImportError: - raise Exception('No address for the PXE server found in data.py (`PXE_CONFIG_SERVER`)') - -def generate_boot_conf(directory, installer, action): - # in case of restore, we disable the text ui from the installer completely, - # to workaround a bug that leaves us stuck on a confirmation dialog at the end of the operation. - rt = 'rt=1' if action == 'restore' else '' - with open(f'{directory}/boot.conf', 'w') as bootfile: - bootfile.write(f"""answerfile=custom -installer={installer} -is_default=1 -{rt} -""") + return pxe.PXE_CONFIG_SERVER def generate_answerfile(directory, installer, hostname_or_ip, target_hostname, action, hdd, netinstall_gpg_check): pxe = pxe_address() @@ -89,37 +73,16 @@ def generate_answerfile(directory, installer, hostname_or_ip, target_hostname, a raise Exception(f"Unknown action: `{action}`") def copy_files_to_pxe(mac_address, tmp_local_path): - assert mac_address - pxe = pxe_address() - remote_dir = f'{PXE_CONFIG_DIR}/{mac_address}/' - clean_files_on_pxe(mac_address) - ssh(pxe, ['mkdir', '-p', remote_dir]) - scp(pxe, f'{tmp_local_path}/boot.conf', remote_dir) - scp(pxe, f'{tmp_local_path}/answerfile.xml', remote_dir) + pxe.server_push_config(mac_address, tmp_local_path) def clean_files_on_pxe(mac_address): - assert mac_address # protection against deleting the whole parent dir! - pxe = pxe_address() - remote_dir = f'{PXE_CONFIG_DIR}/{mac_address}/' - ssh(pxe, ['rm', '-rf', remote_dir]) + pxe.server_remove_config(mac_address) def clean_bootconf_on_pxe(mac_address): - assert mac_address - pxe = pxe_address() - distant_file = f'{PXE_CONFIG_DIR}/{mac_address}/boot.conf' - try: - ssh(pxe, ['rm', '-rf', distant_file]) - except SSHCommandFailed as e: - raise Exception('ERROR: failed to clean the boot.conf file.' + e) + pxe.server_remove_bootconf(mac_address) def get_candidate_ips(mac_address): - pxe = pxe_address() - output = ssh( - pxe, - ['arp', '-n', '|', 'grep', mac_address, '|', 'awk', '\'{ print $1 }\''] - ) - candidate_ips = output.splitlines() - return candidate_ips + return pxe.arp_addresses_for(mac_address) def is_ip_active(ip): return not os.system(f"ping -c 3 -W 10 {ip} > /dev/null 2>&1") @@ -191,8 +154,6 @@ def main(): # *** "fail early" checks - pxe = pxe_address() # raises if not defined - if not is_uuid(args.vm_uuid): raise Exception(f'The provided VM UUID is invalid: {args.vm_uuid}') @@ -237,7 +198,7 @@ def main(): hdd = 'nvme0n1' if vm.is_uefi else 'sda' generate_answerfile(tmp_local_path, installer, args.host, args.target_hostname, args.action, hdd, netinstall_gpg_check) - generate_boot_conf(tmp_local_path, installer, args.action) + pxe.generate_boot_conf(tmp_local_path, installer) logging.info('Copy files to the pxe server') copy_files_to_pxe(mac_address, tmp_local_path) atexit.register(lambda: clean_files_on_pxe(mac_address)) diff --git a/tests/guest-tools/unix/test_guest_tools_unix.py b/tests/guest-tools/unix/test_guest_tools_unix.py index 51664777..4bb82bc0 100644 --- a/tests/guest-tools/unix/test_guest_tools_unix.py +++ b/tests/guest-tools/unix/test_guest_tools_unix.py @@ -9,13 +9,13 @@ # - hostA2: Second member of the pool. Can have any local SR. No need to specify it on CLI. # From --vm parameter # - A VM to import, supported by the Linux/install.sh script of the guest tools ISO +# (without this flag you get an alpine, and that is not suitable) class State: def __init__(self): self.tools_version = None self.vm_distro = None -@pytest.mark.incremental # tests depend on each other. If one test fails, don't execute the others @pytest.mark.multi_vms @pytest.mark.usefixtures("unix_vm") class TestGuestToolsUnix: @@ -33,7 +33,8 @@ def _check_os_info(self, vm, vm_distro): detected_distro = vm.distro() assert detected_distro == vm_distro - def test_install(self, running_vm, state): + @pytest.fixture(scope="class", autouse=True) + def vm_install(self, running_vm, state): vm = running_vm # skip test for some unixes @@ -62,7 +63,7 @@ def test_install(self, running_vm, state): # mount ISO logging.info("Mount guest tools ISO") - vm.mount_guest_tools_iso() + vm.insert_guest_tools_iso() tmp_mnt = vm.ssh(['mktemp', '-d']) time.sleep(2) # wait a small amount of time just to ensure the device is available vm.ssh(['mount', '-t', 'iso9660', '/dev/cdrom', tmp_mnt]) @@ -80,7 +81,7 @@ def test_install(self, running_vm, state): # unmount ISO logging.info("Unmount guest tools ISO") vm.ssh(['umount', tmp_mnt]) - vm.unmount_guest_tools_iso() + vm.eject_cd() # check that xe-daemon is running wait_for(lambda: vm.ssh_with_result(['pgrep', '-f', 'xe-daemon']).returncode == 0, diff --git a/tests/install/__init__.py b/tests/install/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/install/conftest.py b/tests/install/conftest.py new file mode 100644 index 00000000..6c78e8e1 --- /dev/null +++ b/tests/install/conftest.py @@ -0,0 +1,258 @@ +from copy import deepcopy +import importlib +import logging +import os +import pytest +import pytest_dependency +import tempfile +import xml.etree.ElementTree as ET + +from lib.installer import AnswerFile +from lib.common import callable_marker +from lib.commands import local_cmd, scp, ssh + +ISO_IMAGES = getattr(importlib.import_module('data', package=None), 'ISO_IMAGES', {}) + +# Return true if the version of the ISO doesn't support the source type. +# Note: this is a quick-win hack, to avoid explicit enumeration of supported +# source_type values for each ISO. +def skip_source_type(version, source_type): + if version not in ISO_IMAGES.keys(): + return True, "version of ISO {} is unknown".format(version) + + if source_type == "iso": + if ISO_IMAGES[version].get('net-only', False): + return True, "ISO image is net-only while source_type is local" + + return False, "do not skip" + + if source_type == "net": + # Net install is not valid if there is no netinstall URL + # FIXME: ISO includes a default URL so we should be able to omit net-url + if 'net-url' not in ISO_IMAGES[version].keys(): + return True, "net-url required for netinstall was not found for {}".format(version) + + return False, "do not skip" + + # If we don't know the source type then it is invalid + return True, "unknown source type {}".format(source_type) + +@pytest.fixture(scope='function') +def answerfile(request): + marker = request.node.get_closest_marker("answerfile") + + if marker is None: + yield None # no answerfile to generate + return + + # construct answerfile definition from option "base", and explicit bits + param_mapping = marker.kwargs.get("param_mapping", {}) + answerfile_def = callable_marker(marker.args[0], request, param_mapping=param_mapping) + assert isinstance(answerfile_def, AnswerFile) + + from data import HOSTS_IP_CONFIG + answerfile_def.top_append( + dict(TAG="admin-interface", + name="eth0", + proto="static", + CONTENTS=( + dict(TAG='ipaddr', CONTENTS=HOSTS_IP_CONFIG['HOSTS']['DEFAULT']), + dict(TAG='subnet', CONTENTS=HOSTS_IP_CONFIG['NETMASK']), + dict(TAG='gateway', CONTENTS=HOSTS_IP_CONFIG['GATEWAY']), + )), + dict(TAG="name-server", + CONTENTS=HOSTS_IP_CONFIG['DNS']), + ) + + yield answerfile_def + +@pytest.fixture(scope='function') +def iso_remaster(request, answerfile): + marker = request.node.get_closest_marker("installer_iso") + assert marker is not None, "iso_remaster fixture requires 'installer_iso' marker" + param_mapping = marker.kwargs.get("param_mapping", {}) + (iso_key, source_type) = callable_marker(marker.args[0], request, param_mapping=param_mapping) + + try: + source_type = request.getfixturevalue("source_type") + except pytest.FixtureLookupError as e: + raise RuntimeError("iso_remaster fixture requires 'source_type' parameter") from e + + gen_unique_uuid = marker.kwargs.get("gen_unique_uuid", False) + + if source_type == "pxe": + # ISO remastering is not needed when booting with PXE so we just return + logging.info("iso_remaster not needed with PXE") + yield None + return + + skip, reason = skip_source_type(iso_key, source_type) + if skip: + pytest.skip(reason) + + from data import ISOSR_SRV, ISOSR_PATH, PXE_CONFIG_SERVER, TEST_SSH_PUBKEY, TOOLS + assert "iso-remaster" in TOOLS + iso_remaster = TOOLS["iso-remaster"] + assert os.access(iso_remaster, os.X_OK) + + assert iso_key in ISO_IMAGES, f"ISO_IMAGES does not have a value for {iso_key}" + SOURCE_ISO = ISO_IMAGES[iso_key]['path'] + + with tempfile.TemporaryDirectory() as isotmp: + remastered_iso = os.path.join(isotmp, "image.iso") + img_patcher_script = os.path.join(isotmp, "img-patcher") + iso_patcher_script = os.path.join(isotmp, "iso-patcher") + answerfile_xml = os.path.join(isotmp, "answerfile.xml") + + if answerfile: + logging.info("generating answerfile %s", answerfile_xml) + answerfile.top_append(dict(TAG="script", stage="filesystem-populated", + type="url", CONTENTS="file:///root/postinstall.sh")) + answerfile.write_xml(answerfile_xml) + else: + logging.info("no answerfile") + + logging.info("Remastering %s to %s", SOURCE_ISO, remastered_iso) + + # generate install.img-patcher script + with open(img_patcher_script, "xt") as patcher_fd: + print(f"""#!/bin/bash +set -ex +INSTALLIMG="$1" + +mkdir -p "$INSTALLIMG/root/.ssh" +echo "{TEST_SSH_PUBKEY}" > "$INSTALLIMG/root/.ssh/authorized_keys" + +test ! -e "{answerfile_xml}" || + cp "{answerfile_xml}" "$INSTALLIMG/root/answerfile.xml" + +cat > "$INSTALLIMG/usr/local/sbin/test-pingpxe.sh" << 'EOF' +#! /bin/bash +set -eE +set -o pipefail + +ether_of () {{ + ifconfig "$1" | grep ether | sed 's/.*ether \\([^ ]*\\).*/\\1/' +}} + +# on installed system, avoid xapi-project/xen-api#5799 +if ! [ -e /opt/xensource/installer ]; then + eth_mac=$(ether_of eth0) + br_mac=$(ether_of xenbr0) + + # wait for bridge MAC to be fixed + test "$eth_mac" = "$br_mac" +fi + +ping -c1 "$1" +EOF +chmod +x "$INSTALLIMG/usr/local/sbin/test-pingpxe.sh" + +cat > "$INSTALLIMG/etc/systemd/system/test-pingpxe.service" < "$INSTALLIMG/root/test-unique-uuids.service" < "$INSTALLIMG/root/postinstall.sh" <> "\\$ROOT/root/.ssh/authorized_keys" +EOF +""", + file=patcher_fd) + os.chmod(patcher_fd.fileno(), 0o755) + + # generate iso-patcher script + with open(iso_patcher_script, "xt") as patcher_fd: + passwd = "passw0rd" # FIXME use invalid hash + print(f"""#!/bin/bash +set -ex +ISODIR="$1" +SED_COMMANDS=(-e "s@/vmlinuz@/vmlinuz sshpassword={passwd} atexit=shell network_device=all@") +test ! -e "{answerfile_xml}" || + SED_COMMANDS+=(-e "s@/vmlinuz@/vmlinuz install answerfile=file:///root/answerfile.xml@") + +sed -i "${{SED_COMMANDS[@]}}" \ + "$ISODIR"/*/*/grub*.cfg \ + "$ISODIR"/boot/isolinux/isolinux.cfg +""", + file=patcher_fd) + os.chmod(patcher_fd.fileno(), 0o755) + + # do remaster + local_cmd([iso_remaster, + "--install-patcher", img_patcher_script, + "--iso-patcher", iso_patcher_script, + SOURCE_ISO, remastered_iso + ]) + + # unique filename on server, has to work on FreeBSD-based NAS + # too, and even v14 has no tool allowing mktemp suffixes + remote_iso = ssh(ISOSR_SRV, + ["python3", "-c", + '"import os, tempfile; ' + f"f = tempfile.mkstemp(suffix='.iso', dir='{ISOSR_PATH}')[1];" + "os.chmod(f, 0o644);" + 'print(f);"' + ]) + logging.info("Uploading to ISO-SR server remastered %s as %s", + remastered_iso, os.path.basename(remote_iso)) + scp(ISOSR_SRV, remastered_iso, remote_iso) + # FIXME: is sr-scan ever needed? + + try: + yield os.path.basename(remote_iso) + finally: + logging.info("Removing %s from ISO-SR server", os.path.basename(remote_iso)) + ssh(ISOSR_SRV, ["rm", remote_iso]) + +@pytest.fixture(scope='function') +def xcpng_chained(request): + # take test name from mark + marker = request.node.get_closest_marker("continuation_of") + assert marker is not None, "xcpng_chained fixture requires 'continuation_of' marker" + param_mapping = marker.kwargs.get("param_mapping", {}) + continuation_of = callable_marker(marker.args[0], request, param_mapping=param_mapping) + + vm_defs = [dict(name=vm_spec['vm'], + image_test=vm_spec['image_test'], + image_vm=vm_spec.get("image_vm", vm_spec['vm']), + image_scope=vm_spec.get("scope", "module"), + ) + for vm_spec in continuation_of] + + depends = [vm_spec['image_test'] for vm_spec in continuation_of] + pytest_dependency.depends(request, depends) + request.applymarker(pytest.mark.vm_definitions(*vm_defs)) diff --git a/tests/install/test-sequences/82-83-bios-iso-ext.lst b/tests/install/test-sequences/82-83-bios-iso-ext.lst new file mode 100644 index 00000000..5d33bc1a --- /dev/null +++ b/tests/install/test-sequences/82-83-bios-iso-ext.lst @@ -0,0 +1,7 @@ +tests/install/test.py::TestNested::test_install[bios-821.1-iso-ext] +tests/install/test.py::TestNested::test_tune_firstboot[None-bios-821.1-host1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_install[bios-821.1-host1-iso-ext] +tests/install/test.py::TestNested::test_upgrade[bios-821.1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[bios-821.1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_restore[bios-821.1-83rc1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[bios-821.1-83rc1-83rc1-iso-ext] diff --git a/tests/install/test-sequences/82-83-bios-iso-lvm.lst b/tests/install/test-sequences/82-83-bios-iso-lvm.lst new file mode 100644 index 00000000..e02b9f33 --- /dev/null +++ b/tests/install/test-sequences/82-83-bios-iso-lvm.lst @@ -0,0 +1,7 @@ +tests/install/test.py::TestNested::test_install[bios-821.1-iso-lvm] +tests/install/test.py::TestNested::test_tune_firstboot[None-bios-821.1-host1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_install[bios-821.1-host1-iso-lvm] +tests/install/test.py::TestNested::test_upgrade[bios-821.1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_noninst[bios-821.1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_restore[bios-821.1-83rc1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_noninst[bios-821.1-83rc1-83rc1-iso-lvm] diff --git a/tests/install/test-sequences/82-83-rpu.lst b/tests/install/test-sequences/82-83-rpu.lst new file mode 100644 index 00000000..5c1ad22e --- /dev/null +++ b/tests/install/test-sequences/82-83-rpu.lst @@ -0,0 +1,6 @@ +tests/install/test.py::TestNested::test_install[uefi-821.1-iso-nosr] +tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-821.1-host1-iso-nosr] +tests/install/test.py::TestNested::test_firstboot_install[uefi-821.1-host1-iso-nosr] +tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-821.1-host2-iso-nosr] +tests/install/test.py::TestNested::test_firstboot_install[uefi-821.1-host2-iso-nosr] +tests/install/test_pool.py::test_pool_rpu[uefi-821.1-83rc1-iso] diff --git a/tests/install/test-sequences/82-83-uefi-iso-ext.lst b/tests/install/test-sequences/82-83-uefi-iso-ext.lst new file mode 100644 index 00000000..e6c075e5 --- /dev/null +++ b/tests/install/test-sequences/82-83-uefi-iso-ext.lst @@ -0,0 +1,7 @@ +tests/install/test.py::TestNested::test_install[uefi-821.1-iso-ext] +tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-821.1-host1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_install[uefi-821.1-host1-iso-ext] +tests/install/test.py::TestNested::test_upgrade[uefi-821.1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-821.1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_restore[uefi-821.1-83rc1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-821.1-83rc1-83rc1-iso-ext] diff --git a/tests/install/test-sequences/82-83-uefi-iso-lvm.lst b/tests/install/test-sequences/82-83-uefi-iso-lvm.lst new file mode 100644 index 00000000..5b5edf88 --- /dev/null +++ b/tests/install/test-sequences/82-83-uefi-iso-lvm.lst @@ -0,0 +1,7 @@ +tests/install/test.py::TestNested::test_install[uefi-821.1-iso-lvm] +tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-821.1-host1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_install[uefi-821.1-host1-iso-lvm] +tests/install/test.py::TestNested::test_upgrade[uefi-821.1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-821.1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_restore[uefi-821.1-83rc1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-821.1-83rc1-83rc1-iso-lvm] diff --git a/tests/install/test-sequences/83-bios-iso-ext.lst b/tests/install/test-sequences/83-bios-iso-ext.lst new file mode 100644 index 00000000..4b8c2d4b --- /dev/null +++ b/tests/install/test-sequences/83-bios-iso-ext.lst @@ -0,0 +1,7 @@ +tests/install/test.py::TestNested::test_install[bios-83rc1-iso-ext] +tests/install/test.py::TestNested::test_tune_firstboot[None-bios-83rc1-host1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_install[bios-83rc1-host1-iso-ext] +tests/install/test.py::TestNested::test_upgrade[bios-83rc1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[bios-83rc1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_restore[bios-83rc1-83rc1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[bios-83rc1-83rc1-83rc1-iso-ext] diff --git a/tests/install/test-sequences/83-bios-iso-lvm.lst b/tests/install/test-sequences/83-bios-iso-lvm.lst new file mode 100644 index 00000000..e0756d7e --- /dev/null +++ b/tests/install/test-sequences/83-bios-iso-lvm.lst @@ -0,0 +1,7 @@ +tests/install/test.py::TestNested::test_install[bios-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_tune_firstboot[None-bios-83rc1-host1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_install[bios-83rc1-host1-iso-lvm] +tests/install/test.py::TestNested::test_upgrade[bios-83rc1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_noninst[bios-83rc1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_restore[bios-83rc1-83rc1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_noninst[bios-83rc1-83rc1-83rc1-iso-lvm] diff --git a/tests/install/test-sequences/83-uefi-iso-ext.lst b/tests/install/test-sequences/83-uefi-iso-ext.lst new file mode 100644 index 00000000..d79a18c0 --- /dev/null +++ b/tests/install/test-sequences/83-uefi-iso-ext.lst @@ -0,0 +1,7 @@ +tests/install/test.py::TestNested::test_install[uefi-83rc1-iso-ext] +tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-83rc1-host1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_install[uefi-83rc1-host1-iso-ext] +tests/install/test.py::TestNested::test_upgrade[uefi-83rc1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-83rc1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_restore[uefi-83rc1-83rc1-83rc1-iso-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-83rc1-83rc1-83rc1-iso-ext] diff --git a/tests/install/test-sequences/83-uefi-iso-lvm.lst b/tests/install/test-sequences/83-uefi-iso-lvm.lst new file mode 100644 index 00000000..51b383f7 --- /dev/null +++ b/tests/install/test-sequences/83-uefi-iso-lvm.lst @@ -0,0 +1,7 @@ +tests/install/test.py::TestNested::test_install[uefi-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-83rc1-host1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_install[uefi-83rc1-host1-iso-lvm] +tests/install/test.py::TestNested::test_upgrade[uefi-83rc1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-83rc1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_restore[uefi-83rc1-83rc1-83rc1-iso-lvm] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-83rc1-83rc1-83rc1-iso-lvm] diff --git a/tests/install/test-sequences/83-uefi-net-ext.lst b/tests/install/test-sequences/83-uefi-net-ext.lst new file mode 100644 index 00000000..d5814da7 --- /dev/null +++ b/tests/install/test-sequences/83-uefi-net-ext.lst @@ -0,0 +1,7 @@ +tests/install/test.py::TestNested::test_install[uefi-83rc1net-net-ext] +tests/install/test.py::TestNested::test_tune_firstboot[None-uefi-83rc1net-host1-net-ext] +tests/install/test.py::TestNested::test_firstboot_install[uefi-83rc1net-host1-net-ext] +tests/install/test.py::TestNested::test_upgrade[uefi-83rc1net-83rc1net-net-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-83rc1net-83rc1net-net-ext] +tests/install/test.py::TestNested::test_restore[uefi-83rc1net-83rc1net-83rc1net-net-ext] +tests/install/test.py::TestNested::test_firstboot_noninst[uefi-83rc1net-83rc1net-83rc1net-net-ext] diff --git a/tests/install/test.py b/tests/install/test.py new file mode 100644 index 00000000..1431bab7 --- /dev/null +++ b/tests/install/test.py @@ -0,0 +1,492 @@ +import logging +import pytest +import time + +from lib import commands, installer, pxe +from lib.common import safe_split, wait_for, callable_marker +from lib.installer import AnswerFile +from lib.pif import PIF +from lib.pool import Pool +from lib.vdi import VDI + +from data import HOSTS_IP_CONFIG, ISO_IMAGES, NETWORKS +assert "MGMT" in NETWORKS +assert "HOSTS" in HOSTS_IP_CONFIG + +@pytest.mark.dependency() +class TestNested: + @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm")) + @pytest.mark.parametrize("source_type", ("iso", "net", "pxe")) + @pytest.mark.parametrize("iso_version", ( + "83rc1", "83rc1net", "83b2", + "821.1", + "81", "80", "76", "75", + "xs8", "ch821.1", + )) + @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.vm_definitions(lambda firmware: dict( + name="vm1", + template="Other install media", + params=( + # dict(param_name="", value=""), + dict(param_name="memory-static-max", value="4GiB"), + dict(param_name="memory-dynamic-max", value="4GiB"), + dict(param_name="memory-dynamic-min", value="4GiB"), + dict(param_name="platform", key="exp-nested-hvm", value="true"), # FIXME < 8.3 host? + dict(param_name="HVM-boot-params", key="order", value="dcn"), + ) + { + "uefi": ( + dict(param_name="HVM-boot-params", key="firmware", value="uefi"), + dict(param_name="platform", key="device-model", value="qemu-upstream-uefi"), + ), + "bios": (), + }[firmware], + vdis=[dict(name="vm1 system disk", size="100GiB", device="xvda", userdevice="0")], + cd_vbd=dict(device="xvdd", userdevice="3"), + vifs=[dict(index=0, network_uuid=NETWORKS["MGMT"])], + ), + param_mapping={"firmware": "firmware"}) + @pytest.mark.installer_iso( + lambda iso_version, source_type: (iso_version, source_type), + gen_unique_uuid=True, + param_mapping={"iso_version": "iso_version", "source_type": "source_type"}) + @pytest.mark.answerfile(lambda firmware, local_sr, source_type, version: AnswerFile("INSTALL") \ + .top_setattr({} if local_sr == "nosr" else {"sr-type": local_sr}) \ + .top_append( + {'iso': {"TAG": "source", "type": "local"}, + 'net': {"TAG": "source", "type": "url", + # FIXME evaluation requires 'net-url' for every ISO + "CONTENTS": ISO_IMAGES[version]['net-url']}, + 'pxe': {"TAG": "source", "type": "url", + # FIXME evaluation requires 'pxe-url' for every ISO + "CONTENTS": ISO_IMAGES[version]['pxe-url']}}[source_type], + {"TAG": "primary-disk", + "guest-storage": "no" if local_sr == "nosr" else "yes", + "CONTENTS": {"uefi": "nvme0n1", "bios": "sda"}[firmware]}, + ), + param_mapping={"firmware": "firmware", "local_sr": "local_sr", + "source_type": "source_type", "version": "iso_version", + }) + def test_install(self, create_vms, iso_remaster, + firmware, iso_version, source_type, local_sr): + assert len(create_vms) == 1 + + iso_tool = iso_remaster if source_type != "pxe" else None + installer.perform_install(iso=iso_tool, host_vm=create_vms[0], version=iso_version) + + @pytest.fixture + @staticmethod + def helper_vm_with_plugged_disk(imported_vm, create_vms): + helper_vm = imported_vm + host_vm, = create_vms + + helper_vm.start() + helper_vm.wait_for_vm_running_and_ssh_up() + + all_vdis = [VDI(uuid, host=host_vm.host) for uuid in host_vm.vdi_uuids()] + disk_vdis = [vdi for vdi in all_vdis if not vdi.readonly()] + vdi, = disk_vdis + + vbd = helper_vm.create_vbd("1", vdi.uuid) + try: + vbd.plug() + + yield helper_vm + + finally: + vbd.unplug() + vbd.destroy() + + @pytest.mark.usefixtures("xcpng_chained") + @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm")) + @pytest.mark.parametrize("source_type", ("iso", "net")) + @pytest.mark.parametrize("machine", ("host1", "host2")) + @pytest.mark.parametrize("version", ( + "83rc1", + "83b2", + "821.1", + "81", "80", + "76", "75", + "ch821.1", "xs8", + )) + @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.continuation_of(lambda version, firmware, local_sr, source_type: [dict( + vm="vm1", + image_test=f"TestNested::test_install[{firmware}-{version}-{source_type}-{local_sr}]")], + param_mapping={"version": "version", "firmware": "firmware", + "local_sr": "local_sr", "source_type": "source_type"}) + @pytest.mark.small_vm + def test_tune_firstboot(self, create_vms, helper_vm_with_plugged_disk, + firmware, version, machine, local_sr, source_type): + helper_vm = helper_vm_with_plugged_disk + + helper_vm.ssh(["mount /dev/xvdb1 /mnt"]) + try: + # hostname + logging.info("Setting hostname to %r", machine) + helper_vm.ssh(["echo > /mnt/etc/hostname", machine]) + # management IP + if machine in HOSTS_IP_CONFIG['HOSTS']: + ip = HOSTS_IP_CONFIG['HOSTS'][machine] + logging.info("Changing IP to %s", ip) + + helper_vm.ssh([f"sed -i s/^IP=.*/IP='{ip}'/", + "/mnt/etc/firstboot.d/data/management.conf"]) + finally: + helper_vm.ssh(["umount /dev/xvdb1"]) + + def _test_firstboot(self, host_vm, mode, *, machine='DEFAULT'): + vif = host_vm.vifs()[0] + mac_address = vif.param_get('MAC') + logging.info("Host VM has MAC %s", mac_address) + + # determine version info from `mode` + if mode.startswith("xs"): + expected_dist = "XenServer" + elif mode.startswith("ch"): + expected_dist = "CitrixHypervisor" + else: + expected_dist = "XCP-ng" + # succession of insta/upg/rst operations + split_mode = mode.split("-") + if len(split_mode) == 3: + # restore: back to 1st installed version + expected_rel_id = split_mode[0] + else: + expected_rel_id = split_mode[-1] + expected_rel = { + "ch821.1": "8.2.1", + "xs8": "8.4.0", + "75": "7.5.0", + "76": "7.6.0", + "80": "8.0.0", + "81": "8.1.0", + "821.1": "8.2.1", + "83b2": "8.3.0", + "83rc1": "8.3.0", + "83rc1net": "8.3.0", + }[expected_rel_id] + + try: + host_vm.start() + wait_for(host_vm.is_running, "Wait for host VM running") + + host_vm.ip = HOSTS_IP_CONFIG['HOSTS'].get(machine, + HOSTS_IP_CONFIG['HOSTS']['DEFAULT']) + logging.info("Expecting host VM to have IP %s", host_vm.ip) + + wait_for( + lambda: commands.local_cmd( + ["nc", "-zw5", host_vm.ip, "22"], check=False).returncode == 0, + "Wait for ssh back up on Host VM", retry_delay_secs=5, timeout_secs=4 * 60) + + logging.info("Checking installed version") + lsb_dist = commands.ssh(host_vm.ip, ["lsb_release", "-si"]) + lsb_rel = commands.ssh(host_vm.ip, ["lsb_release", "-sr"]) + assert (lsb_dist, lsb_rel) == (expected_dist, expected_rel) + + lsb_rel_tuple = tuple(int(x) for x in lsb_rel.split(".")) + + # wait for XAPI startup to be done, which avoids: + # - waiting for XAPI to start listening to its socket + # - waiting for host and pool objects to be populated after install + wait_for(lambda: commands.ssh(host_vm.ip, ['xapi-wait-init-complete', '60'], + check=False, simple_output=False).returncode == 0, + "Wait for XAPI init to be complete", + timeout_secs=30 * 60) + # FIXME: after this all wait_for should be instant - replace with immediate tests? + + # pool master must be reachable here + pool = Pool(host_vm.ip) + logging.info("Host uuid: %s", pool.master.uuid) + + # wait for XAPI + wait_for(pool.master.is_enabled, "Wait for XAPI to be ready", timeout_secs=30 * 60) + + if lsb_rel in ["8.2.1", "8.3.0", "8.4.0"]: + SERVICES =["control-domain-params-init", + "network-init", + "storage-init", + "generate-iscsi-iqn", + "create-guest-templates", + ] + STAMPS_DIR = "/var/lib/misc" + STAMPS = [f"ran-{service}" for service in SERVICES] + elif lsb_rel in ["7.5.0", "7.6.0", "8.0.0", "8.1.0"]: + SERVICES = ["xs-firstboot"] + STAMPS_DIR = "/etc/firstboot.d/state" + STAMPS = [ + "05-prepare-networking", + "10-prepare-storage", + "15-set-default-storage", + "20-udev-storage", + "25-multipath", + "40-generate-iscsi-iqn", + "50-prepare-control-domain-params", + "60-import-keys", + "60-upgrade-likewise-to-pbis", + "62-create-guest-templates", + "90-flush-pool-db", + "95-legacy-logrotate", + "99-remove-firstboot-flag", + ] + if lsb_rel in ["8.0.0", "8.1.0"]: + STAMPS += [ + "80-common-criteria", + ] + # check for firstboot issues + # FIXME: flaky, must check logs extraction on failure + try: + for stamp in STAMPS: + wait_for(lambda: pool.master.ssh(["test", "-e", f"{STAMPS_DIR}/{stamp}"], + check=False, simple_output=False, + ).returncode == 0, + f"Wait for {stamp} stamp") + except TimeoutError: + logging.warning("investigating lack of %s service stamp", stamp) + for service in SERVICES: + out = pool.master.ssh(["systemctl", "status", service], check=False) + logging.warning("service status: %s", out) + out = pool.master.ssh(["grep", "-r", service, "/var/log"], check=False) + logging.warning("in logs: %s", out) + raise + + if lsb_rel_tuple >= (8, 3): + # Those certs take time to get generated in install case, + # so make sure they are. On upgrade they must be + # preserved (fails on 8.3.0-rc1). + for certfile in ( + "/etc/stunnel/xapi-stunnel-ca-bundle.pem", + "/etc/stunnel/xapi-pool-ca-bundle.pem", + "/etc/xensource/xapi-pool-tls.pem", + ): + wait_for(lambda: pool.master.ssh(["test", "-e", certfile], + check=False, simple_output=False, + ).returncode == 0, + f"Wait for {certfile} certificate file") + + #wait_for(lambda: False, 'Wait "forever"', timeout_secs=100*60) + logging.info("Powering off pool master") + try: + # use "poweroff" because "reboot" would cause ARP and + # SSH to be checked before host is down, and require + # ssh retries + pool.master.ssh(["poweroff"]) + except commands.SSHCommandFailed as e: + # ignore connection closed by reboot + if e.returncode == 255 and "closed by remote host" in e.stdout: + logging.info("sshd closed the connection") + pass + else: + raise + + wait_for(host_vm.is_halted, "Wait for host VM halted") + + except Exception as e: + logging.critical("caught exception %s", e) + # wait_for(lambda: False, 'Wait "forever"', timeout_secs=100*60) + #host_vm.shutdown(force=True) + raise + except KeyboardInterrupt: + logging.warning("keyboard interrupt") + # wait_for(lambda: False, 'Wait "forever"', timeout_secs=100*60) + #host_vm.shutdown(force=True) + raise + + @pytest.mark.usefixtures("xcpng_chained") + @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm")) + @pytest.mark.parametrize("source_type", ("iso", "net")) + @pytest.mark.parametrize("machine", ("host1", "host2")) + @pytest.mark.parametrize("version", ( + "83rc1", "83rc1net", + "83b2", + "821.1", + "81", "80", + "76", "75", + "ch821.1", "xs8", + )) + @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.continuation_of(lambda version, firmware, machine, local_sr, source_type: [dict( + vm="vm1", + image_test=f"TestNested::test_tune_firstboot[None-{firmware}-{version}-{machine}-{source_type}-{local_sr}]")], + param_mapping={"version": "version", "firmware": "firmware", + "machine": "machine", "local_sr": "local_sr", + "source_type": "source_type"}) + def test_firstboot_install(self, create_vms, + firmware, version, machine, source_type, local_sr): + host_vm = create_vms[0] + + self._test_firstboot(host_vm, version, machine=machine) + + @pytest.mark.usefixtures("xcpng_chained") + @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm")) + @pytest.mark.parametrize("source_type", ("iso", "net")) + @pytest.mark.parametrize("mode", ( + "83rc1-83rc1", "83rc1-83rc1-83rc1", + "83rc1net-83rc1net", "83rc1net-83rc1net-83rc1net", + "83b2-83rc1", + "821.1-83rc1", + "821.1-83rc1-83rc1", + "81-83rc1", "81-83rc1-83rc1", + "80-83rc1", "80-83rc1-83rc1", + "76-83rc1", "76-83rc1-83rc1", + "75-83rc1", "75-83rc1-83rc1", + "ch821.1-83rc1", + "ch821.1-83rc1-83rc1", + "821.1-821.1", + )) + @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.continuation_of(lambda params, firmware, local_sr, source_type: [dict( + vm="vm1", + image_test=(f"TestNested::{{}}[{firmware}-{params}-{source_type}-{local_sr}]".format( + { + 2: "test_upgrade", + 3: "test_restore", + }[len(params.split("-"))] + )))], + param_mapping={"params": "mode", "firmware": "firmware", + "local_sr": "local_sr", "source_type": "source_type"}) + def test_firstboot_noninst(self, create_vms, + firmware, mode, source_type, local_sr): + host_vm = create_vms[0] + self._test_firstboot(host_vm, mode) + + @pytest.mark.usefixtures("xcpng_chained") + @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm")) + @pytest.mark.parametrize("source_type", ("iso", "net")) + @pytest.mark.parametrize(("orig_version", "iso_version"), [ + ("83rc1", "83rc1"), + ("83rc1net", "83rc1net"), + ("83b2", "83rc1"), + ("821.1", "83rc1"), + ("81", "83rc1"), + ("80", "83rc1"), + ("76", "83rc1"), + ("75", "83rc1"), + ("ch821.1", "83rc1"), + ("821.1", "821.1"), + ]) + @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.continuation_of(lambda firmware, params, local_sr, source_type: [dict( + vm="vm1", + image_test=f"TestNested::test_firstboot_install[{firmware}-{params}-host1-{source_type}-{local_sr}]")], + param_mapping={"params": "orig_version", "firmware": "firmware", + "local_sr": "local_sr", "source_type": "source_type"}) + @pytest.mark.installer_iso( + lambda iso_version, source_type: (iso_version, source_type), + param_mapping={"iso_version": "iso_version", "source_type": "source_type"}) + @pytest.mark.answerfile( + lambda firmware: AnswerFile("UPGRADE").top_append( + {"TAG": "source", "type": "local"}, + {"TAG": "existing-installation", + "CONTENTS": {"uefi": "nvme0n1", "bios": "sda"}[firmware]}, + ), + param_mapping={"firmware": "firmware"}) + def test_upgrade(self, create_vms, iso_remaster, + firmware, orig_version, iso_version, source_type, local_sr): + installer.perform_upgrade(iso=iso_remaster, host_vm=create_vms[0]) + + @pytest.mark.usefixtures("xcpng_chained") + @pytest.mark.parametrize("local_sr", ("nosr", "ext", "lvm")) + @pytest.mark.parametrize("source_type", ("iso", "net")) + @pytest.mark.parametrize(("orig_version", "iso_version"), [ + ("83rc1-83rc1", "83rc1"), + ("83rc1net-83rc1net", "83rc1net"), + ("821.1-83rc1", "83rc1"), + ("75-83rc1", "83rc1"), + ("76-83rc1", "83rc1"), + ("80-83rc1", "83rc1"), + ("81-83rc1", "83rc1"), + ("ch821.1-83rc1", "83rc1"), + ]) + @pytest.mark.parametrize("firmware", ("uefi", "bios")) + @pytest.mark.continuation_of(lambda firmware, params, local_sr, source_type: [dict( + vm="vm1", + image_test=f"TestNested::test_firstboot_noninst[{firmware}-{params}-{source_type}-{local_sr}]")], + param_mapping={"params": "orig_version", "firmware": "firmware", + "local_sr": "local_sr", "source_type": "source_type"}) + @pytest.mark.installer_iso( + lambda iso_version, source_type: (iso_version, source_type), + param_mapping={"iso_version": "iso_version", "source_type": "source_type"}) + @pytest.mark.answerfile(lambda firmware: AnswerFile("RESTORE").top_append( + {"TAG": "backup-disk", + "CONTENTS": {"uefi": "nvme0n1", "bios": "sda"}[firmware]}, + ), + param_mapping={"firmware": "firmware"}) + def test_restore(self, create_vms, iso_remaster, + firmware, orig_version, iso_version, source_type, local_sr): + host_vm = create_vms[0] + vif = host_vm.vifs()[0] + mac_address = vif.param_get('MAC') + logging.info("Host VM has MAC %s", mac_address) + + host_vm.insert_cd(iso_remaster) + + try: + host_vm.start() + wait_for(host_vm.is_running, "Wait for host VM running") + + # catch host-vm IP address + wait_for(lambda: pxe.arp_addresses_for(mac_address), + "Wait for DHCP server to see Host VM in ARP tables", + timeout_secs=10*60) + ips = pxe.arp_addresses_for(mac_address) + logging.info("Host VM has IPs %s", ips) + assert len(ips) == 1 + host_vm.ip = ips[0] + + # wait for "yum install" phase to start + wait_for(lambda: host_vm.ssh(["grep", + "'Restoring backup'", + "/tmp/install-log"], + check=False, simple_output=False, + ).returncode == 0, + "Wait for data restoration to start", + timeout_secs=40*60) # FIXME too big + + # wait for "yum install" phase to finish + wait_for(lambda: host_vm.ssh(["grep", + "'Data restoration complete. About to re-install bootloader.'", + "/tmp/install-log"], + check=False, simple_output=False, + ).returncode == 0, + "Wait for data restoration to complete", + timeout_secs=40*60) # FIXME too big + + # The installer will not terminate in restore mode, it + # requires human interaction and does not even log it, so + # wait for last known action log (tested with 8.3b2) + wait_for(lambda: host_vm.ssh(["grep", + "'ran .*swaplabel.*rc 0'", + "/tmp/install-log"], + check=False, simple_output=False, + ).returncode == 0, + "Wait for installer to hopefully finish", + timeout_secs=40*60) # FIXME too big + + # "wait a bit to be extra sure". Yuck. + time.sleep(30) + + logging.info("Shutting down Host VM after successful restore") + try: + host_vm.ssh(["poweroff"]) + except commands.SSHCommandFailed as e: + # ignore connection closed by reboot + if e.returncode == 255 and "closed by remote host" in e.stdout: + logging.info("sshd closed the connection") + pass + else: + raise + wait_for(host_vm.is_halted, "Wait for host VM halted") + host_vm.eject_cd() + + except Exception as e: + logging.critical("caught exception %s", e) + # wait_for(lambda: False, 'Wait "forever"', timeout_secs=100*60) + #host_vm.shutdown(force=True) + raise + except KeyboardInterrupt: + logging.warning("keyboard interrupt") + # wait_for(lambda: False, 'Wait "forever"', timeout_secs=100*60) + #host_vm.shutdown(force=True) + raise diff --git a/tests/install/test_fixtures.py b/tests/install/test_fixtures.py new file mode 100644 index 00000000..d38c59dd --- /dev/null +++ b/tests/install/test_fixtures.py @@ -0,0 +1,61 @@ +import logging +import os +import pytest + +from lib.common import wait_for +from lib.installer import AnswerFile +from lib.vdi import VDI + +# test the answerfile fixture can run on 2 parametrized instances +# of the test in one run +@pytest.mark.answerfile(lambda: AnswerFile("INSTALL").top_append( + {"TAG": "source", "type": "local"}, + {"TAG": "primary-disk", "text": "nvme0n1"}, +)) +@pytest.mark.parametrize("parm", [ + 1, + pytest.param(2, marks=[ + pytest.mark.dependency(depends=["TestFixtures::test_parametrized_answerfile[1]"]), + ]), +]) +@pytest.mark.dependency +def test_parametrized_answerfile(answerfile, parm): + logging.debug("test_parametrized_answerfile with parm=%s", parm) + +@pytest.fixture +def cloned_disk(imported_vm): + vm = imported_vm + all_vdis = [VDI(uuid, host=vm.host) for uuid in vm.vdi_uuids()] + disk_vdis = [vdi for vdi in all_vdis if not vdi.readonly()] + base_vdi, = disk_vdis + + clone = base_vdi.clone() + yield clone + + clone.destroy() + +@pytest.fixture +def vm_with_plugged_disk(imported_vm, cloned_disk): + vm = imported_vm + test_vdi = cloned_disk + + vm.start() + vm.wait_for_vm_running_and_ssh_up() + + vbd = vm.create_vbd("1", test_vdi.uuid) + try: + vbd.plug() + + yield vm + + finally: + logging.info("cleaning up") + vbd.unplug() + vbd.destroy() + +@pytest.mark.small_vm +def test_vdi_modify(vm_with_plugged_disk): + vm = vm_with_plugged_disk + vm.ssh(["mount /dev/xvdb3 /mnt"]) + vm.ssh(["touch /tmp/foo"]) + vm.ssh(["umount /dev/xvdb3"]) diff --git a/tests/install/test_pool.py b/tests/install/test_pool.py new file mode 100644 index 00000000..af164031 --- /dev/null +++ b/tests/install/test_pool.py @@ -0,0 +1,165 @@ +import logging +import os +import pytest + +from lib import commands, installer, pxe +from lib.common import wait_for, vm_image +from lib.installer import AnswerFile +from lib.pool import Pool + +from data import HOSTS_IP_CONFIG, NFS_DEVICE_CONFIG + +# FIXME without --ignore-unknown-dependency, SKIPPED +# "because it depends on tests/install/test.py::TestNested::test_firstboot_install[uefi-821.1-host1]" +@pytest.mark.usefixtures("xcpng_chained") +@pytest.mark.parametrize("source_type", ("iso", "net")) +@pytest.mark.parametrize(("orig_version", "iso_version"), [ + ("821.1", "83rc1"), +]) +@pytest.mark.parametrize("firmware", ("uefi", "bios")) +@pytest.mark.continuation_of(lambda params, firmware: [ + dict(vm="vm1", + image_test=f"tests/install/test.py::TestNested::test_firstboot_install[{firmware}-{params}-host1-iso-nosr]", + scope="session"), + dict(vm="vm2", + image_vm="vm1", + image_test=f"tests/install/test.py::TestNested::test_firstboot_install[{firmware}-{params}-host2-iso-nosr]", + scope="session"), +], + param_mapping={"params": "orig_version", "firmware": "firmware"}) +@pytest.mark.installer_iso( + lambda version: { + "83rc1": "xcpng-8.3-rc1", + }[version], + param_mapping={"version": "iso_version"}) +@pytest.mark.answerfile( + lambda firmware: AnswerFile("UPGRADE").top_append( + {"TAG": "source", "type": "local"}, + {"TAG": "existing-installation", "CONTENTS": {"uefi": "nvme0n1", "bios": "sda"}[firmware]}, + ), + param_mapping={"firmware": "firmware"}) +def test_pool_rpu(create_vms, iso_remaster, + firmware, orig_version, iso_version, source_type): + (master_vm, slave_vm) = create_vms + master_mac = master_vm.vifs()[0].param_get('MAC') + logging.info("Master VM has MAC %s", master_mac) + slave_mac = slave_vm.vifs()[0].param_get('MAC') + logging.info("Slave VM has MAC %s", slave_mac) + + master_vm.start() + slave_vm.start() + wait_for(master_vm.is_running, "Wait for master VM running") + wait_for(slave_vm.is_running, "Wait for slave VM running") + + master_vm.ip = HOSTS_IP_CONFIG['HOSTS']['DEFAULT'] + logging.info("Expecting master VM to have IP %s", master_vm.ip) + + slave_vm.ip = HOSTS_IP_CONFIG['HOSTS']['host2'] + logging.info("Expecting slave VM to have IP %s", slave_vm.ip) + + wait_for(lambda: not os.system(f"nc -zw5 {master_vm.ip} 22"), + "Wait for ssh up on Master VM", retry_delay_secs=5) + wait_for(lambda: not os.system(f"nc -zw5 {slave_vm.ip} 22"), + "Wait for ssh up on Slave VM", retry_delay_secs=5) + + wait_for(lambda: commands.ssh(master_vm.ip, ['xapi-wait-init-complete', '60'], + check=False, simple_output=False).returncode == 0, + "Wait for master XAPI init to be complete", + timeout_secs=30 * 60) + pool = Pool(master_vm.ip) + + # create pool with shared SR + + wait_for(lambda: commands.ssh(slave_vm.ip, ['xapi-wait-init-complete', '60'], + check=False, simple_output=False).returncode == 0, + "Wait for slave XAPI init to be complete", + timeout_secs=30 * 60) + slave = Pool(slave_vm.ip).master + slave.join_pool(pool) + + sr = pool.master.sr_create("nfs", "NFS Shared SR", NFS_DEVICE_CONFIG, + shared=True, verify=True) + + # create and start VMs + vms = ( + pool.master.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid), + pool.master.import_vm(vm_image('mini-linux-x86_64-bios'), sr_uuid=sr.uuid), + ) + + for vm in vms: + vm.start() + + wait_for(lambda: all(vm.is_running() for vm in vms), "Wait for VMs running") + wait_for(lambda: all(vm.try_get_and_store_ip() for vm in vms), + "Wait for VM IPs", timeout_secs=5*60) + wait_for(lambda: all(vm.is_management_agent_up() for vm in vms), + "Wait for management agents up") + + logging.info("VMs dispatched as %s", [vm.get_residence_host().uuid for vm in vms]) + + ## do RPU + + # evacuate master + vms_to_migrate = [vm for vm in vms if vm.get_residence_host().uuid == pool.master.uuid] + logging.info("Expecting migration of %s", ([vm.uuid for vm in vms_to_migrate],)) + pool.master.xe("host-evacuate", {"host": pool.master.uuid}) + wait_for(lambda: all(vm.get_residence_host().uuid != pool.master.uuid for vm in vms_to_migrate), + "Wait for VM migration") + + # upgrade master + pool.master.shutdown() + wait_for(lambda: master_vm.is_halted(), "Wait for Master VM to be halted", timeout_secs=5*60) + installer.perform_upgrade(iso=iso_remaster, host_vm=master_vm) + pxe.arp_clear_for(master_mac) + master_vm.start() + wait_for(master_vm.is_running, "Wait for Master VM running") + + wait_for(lambda: pxe.arp_addresses_for(master_mac), + "Wait for DHCP server to see Master VM in ARP tables", + timeout_secs=10*60) + ips = pxe.arp_addresses_for(master_mac) + logging.info("Master VM has IPs %s", ips) + assert len(ips) == 1 + master_vm.ip = ips[0] + + wait_for(lambda: not os.system(f"nc -zw5 {master_vm.ip} 22"), + "Wait for ssh back up on Master VM", retry_delay_secs=5) + wait_for(pool.master.is_enabled, "Wait for XAPI to be ready", timeout_secs=30 * 60) + + # evacuate slave + vms_to_migrate = vms + logging.info("Expecting migration of %s", ([vm.uuid for vm in vms_to_migrate],)) + pool.master.xe("host-evacuate", {"host": slave.uuid}) + wait_for(lambda: all(vm.get_residence_host().uuid != slave.uuid for vm in vms), + "Wait for VM migration") + + # upgrade slave + slave.shutdown() + wait_for(lambda: slave_vm.is_halted(), "Wait for Slave VM to be halted", timeout_secs=5*60) + installer.perform_upgrade(iso=iso_remaster, host_vm=slave_vm) + pxe.arp_clear_for(slave_mac) + slave_vm.start() + wait_for(slave_vm.is_running, "Wait for Slave VM running") + + wait_for(lambda: pxe.arp_addresses_for(slave_mac), + "Wait for DHCP server to see Slave VM in ARP tables", + timeout_secs=10*60) + ips = pxe.arp_addresses_for(slave_mac) + logging.info("Slave VM has IPs %s", ips) + assert len(ips) == 1 + slave_vm.ip = ips[0] + + wait_for(lambda: not os.system(f"nc -zw5 {slave_vm.ip} 22"), + "Wait for ssh back up on Slave VM", retry_delay_secs=5) + wait_for(slave.is_enabled, "Wait for XAPI to be ready", timeout_secs=30 * 60) + + logging.info("Migrating a VM back to slave") + vms[1].migrate(slave) + + ## cleanup + + slave.shutdown() + pool.master.shutdown() + wait_for(lambda: slave_vm.is_halted(), "Wait for Slave VM to be halted", timeout_secs=5*60) + wait_for(lambda: master_vm.is_halted(), "Wait for Master VM to be halted", timeout_secs=5*60) + # FIXME destroy shared SR contents diff --git a/tests/misc/test_basic_without_ssh.py b/tests/misc/test_basic_without_ssh.py index 52587df7..de291561 100644 --- a/tests/misc/test_basic_without_ssh.py +++ b/tests/misc/test_basic_without_ssh.py @@ -9,15 +9,14 @@ # because the VM may not have SSH installed, which is needed for more advanced scenarios. # # Requirements: -# - a two-host XCP-ng pool >= 8.1. -# - the pool must have 1 shared SR +# - XCP-ng >= 8.1. +# - a two-host pool with 1 shared SR (for test_live_migrate) +# - the pool must have `suspend-image-SR` set (for suspend and checkpoint) # - each host must have a local SR # - any VM with guest tools installed. No SSH required for this test suite. # - when using an existing VM, the VM can be on any host of the pool, # the local SR or shared SR: the test will adapt itself. # Note however that an existing VM will be left on a different SR after the tests. -# -# This test suite is meant to be run entirely. There's no guarantee that cherry-picking tests will work. @pytest.fixture(scope='session') def existing_shared_sr(host): @@ -25,21 +24,25 @@ def existing_shared_sr(host): assert sr is not None, "A shared SR on the pool is required" return sr -@pytest.mark.incremental # tests depend on each other. If one test fails, don't execute the others @pytest.mark.multi_vms # run them on a variety of VMs @pytest.mark.big_vm # and also on a really big VM ideally -class TestBasicNoSSH: - def test_start(self, imported_vm): - vm = imported_vm - # if VM already running, stop it - if (vm.is_running()): - logging.info("VM already running, shutting it down first") - vm.shutdown(verify=True) - vm.start() - # this also tests the guest tools at the same time since they are used - # for retrieving the IP address and management agent status. - vm.wait_for_os_booted() +def test_vm_start_stop(imported_vm): + vm = imported_vm + # if VM already running, stop it + if (vm.is_running()): + logging.info("VM already running, shutting it down first") + vm.shutdown(verify=True) + vm.start() + # this also tests the guest tools at the same time since they are used + # for retrieving the IP address and management agent status. + vm.wait_for_os_booted() + + vm.shutdown(verify=True) +@pytest.mark.multi_vms # run them on a variety of VMs +@pytest.mark.big_vm # and also on a really big VM ideally +@pytest.mark.usefixtures("started_vm") +class TestBasicNoSSH: def test_pause(self, imported_vm): vm = imported_vm vm.pause(verify=True) @@ -104,7 +107,3 @@ def live_migrate(vm, dest_host, dest_sr, check_vdis=False): else: logging.info("* Preparing for live migration without storage motion *") live_migrate(vm, host1, existing_shared_sr) - - def test_shutdown(self, imported_vm): - vm = imported_vm - vm.shutdown(verify=True)