diff --git a/changelog.md b/changelog.md index 6bf8a1e..479207d 100644 --- a/changelog.md +++ b/changelog.md @@ -2,6 +2,26 @@ This page was created to track changes to versions of Python-ESXi-Utilities (esxi_utils). The changelog was created in v3.22.1 and only changes starting from that version are tracked here. +## 4.0.0 + +- Changes the default method of retrieving VirtualMachine objects from their list + - From: Being scoped just to the 'child' server in a vCenter + - To: All available virtual machines as visible via the vCenter inventory + - The old method of getting a list of virtual machines is available by specifying legacy_list=True to the ESXiClient object when creating it + +- Adds support for Virtual Machine 'Templates' in vCenter arrangements + - vm.is_template() for determining if a VM has been converted to a template + - vm.to_template() to convert a VM to a clonable template (THIS CANNOT BE UNDONE) + - vm.deploy_from_template(...) to create a new VM from a VM template +- Adds a new client.is_vcenter() method to the ESXi client object +- Adds new properties to VirtualMachine object for determining their runtime (child) ESXi host: + - vm.host_system reveals the vim.HostSystem + - vm.esxi_host_name reveals the 'hostname' of the ESXi (child) server owning this VM's resources +- Adds new parameters (host and resource_pool) to vm.create and vm.upload to control which ESXi (child) host and resource pool is deployed to (default is still the legacy operation of 'the currently connected host') +- VirtualMachine to 'str' now attempts to show the runtime host instead of the connected child server in vCenter +- Fix several regex warnings from this library in Python version 3.12+ by adding the 'raw string' character in front of the regex strings +- Silences the deprecation warning coming from this library about the pinning of setuptools pkg_resources API + ## 3.22.1 - Adds metadata to pip package for PyPI diff --git a/esxi_utils/client.py b/esxi_utils/client.py index b3aa81a..c0cfa61 100644 --- a/esxi_utils/client.py +++ b/esxi_utils/client.py @@ -31,8 +31,10 @@ class ESXiClient: :param child_hostname: When connecting to a vCenter instance, the hostname or IP of the child ESXi server. :param child_username: When connecting to a vCenter instance, the username to login to the child ESXi server (for SSH). :param child_password: When connecting to a vCenter instance, the password of the child ESXi user. + :param use_legacy_vm_list: When collecting all Virtual Machines from the ESXi server, this setting decides whether the VMs collected are only from the child server on vCenter (True) or if the VM list contains all VMs as seen via the vCenter inventory (False). """ - def __init__(self, hostname: str, username: str, password: str, child_hostname: typing.Optional[str] = None, child_username: typing.Optional[str] = None, child_password: typing.Optional[str] = None): + def __init__(self, hostname: str, username: str, password: str, child_hostname: typing.Optional[str] = None, child_username: typing.Optional[str] = None, child_password: typing.Optional[str] = None, use_legacy_vm_list: bool = False): + self.use_legacy_vm_list = use_legacy_vm_list context = ssl.SSLContext(ssl.PROTOCOL_SSLv23) context.verify_mode = ssl.CERT_NONE err = None @@ -117,7 +119,7 @@ def vms(self) -> 'VirtualMachineList': """ The Virtual Machine handler for this host. """ - return VirtualMachineList(self) + return VirtualMachineList(self, container=None, legacy_list=self.use_legacy_vm_list) @property def firewall(self) -> 'Firewall': @@ -280,6 +282,12 @@ def __repr__(self): def __del__(self): self.close() + def _content(self): + return self._service_instance.RetrieveContent() + + def is_vcenter(self) -> 'bool': + return str(self._content().about.apiType).lower() == "virtualcenter" + def _get_vim_objects_from(self, root, vim_type): """ Search the ESXi server for the all instances of the specified vim object under the given root object. @@ -291,10 +299,11 @@ def _get_vim_objects_from(self, root, vim_type): """ if not isinstance(vim_type, list): vim_type = [vim_type] - content = self._service_instance.RetrieveContent() - container = content.viewManager.CreateContainerView(root, vim_type, True) - objs = [ ref for ref in container.view ] - container.Destroy() + try: + container = self._content().viewManager.CreateContainerView(root, vim_type, True) + objs = [ ref for ref in container.view ] + finally: + container.Destroy() return objs def _get_vim_objects(self, vim_type, query_root: bool = False): @@ -309,7 +318,7 @@ def _get_vim_objects(self, vim_type, query_root: bool = False): # request only content based on the currently connected 'host_system' server if not query_root and self._child_hostname and self._host_system: return self._get_vim_objects_from(self._host_system, vim_type) - return self._get_vim_objects_from(self._service_instance.RetrieveContent().rootFolder, vim_type) + return self._get_vim_objects_from(self._content().rootFolder, vim_type) def _get_vim_object(self, vim_type, name: str): """ diff --git a/esxi_utils/datastore.py b/esxi_utils/datastore.py index b263d8a..0f3db41 100644 --- a/esxi_utils/datastore.py +++ b/esxi_utils/datastore.py @@ -841,7 +841,7 @@ def parse(client: 'ESXiClient', filepath: str) -> 'DatastoreFile': if filepath.startswith("["): try: - match = re.match("\[(" + "|".join([ re.escape(name) for name in client.datastores.names ]) + ")\]\s*(.+)", filepath, flags=re.IGNORECASE) + match = re.match(r"\[(" + "|".join([ re.escape(name) for name in client.datastores.names ]) + r")\]\s*(.+)", filepath, flags=re.IGNORECASE) datastore_name = match.group(1) relative_path = match.group(2).lstrip("/") return DatastoreFile(client.datastores.get(datastore_name), relative_path) diff --git a/esxi_utils/firewall/firewall.py b/esxi_utils/firewall/firewall.py index 67fc112..d278a42 100644 --- a/esxi_utils/firewall/firewall.py +++ b/esxi_utils/firewall/firewall.py @@ -99,7 +99,7 @@ def _update_service_xml(self, new_tree): :param new_tree: The `etree` XML object to write to the service file. """ new_content = etree.tostring(new_tree, pretty_print=True, encoding='unicode') - new_content = re.sub(r"", "\g<0>\n", new_content, flags=re.MULTILINE) + new_content = re.sub(r"", r"\g<0>\n", new_content, flags=re.MULTILINE) with self._client.ssh() as conn: err = None try: diff --git a/esxi_utils/util/connect/cisco.py b/esxi_utils/util/connect/cisco.py index c94a97e..af6978d 100644 --- a/esxi_utils/util/connect/cisco.py +++ b/esxi_utils/util/connect/cisco.py @@ -1,5 +1,14 @@ from esxi_utils.util.connect.ssh import SSHConnection from dateutil.parser import parse as parsetime + +# Silence the deprecation warning for pkg_resources for now +import warnings +warnings.filterwarnings( + "ignore", + category=UserWarning, + message=r".*pkg_resources is deprecated as an API.*", +) + import pkg_resources import datetime import textfsm @@ -173,7 +182,7 @@ def _parse_table(self, stringtable, headers): """ # Remove empty lines and whitespace lines lines = list(filter(lambda str: str and not str.isspace(),stringtable.splitlines())) - regex = re.compile('\s*'+'\s*'.join([re.escape(head) for head in headers])+'\s*') + regex = re.compile(r'\s*'+r'\s*'.join([re.escape(head) for head in headers])+r'\s*') # Find the header of the table while lines and not regex.match(lines[0]): diff --git a/esxi_utils/util/connect/panos.py b/esxi_utils/util/connect/panos.py index b2ba7af..2461b97 100644 --- a/esxi_utils/util/connect/panos.py +++ b/esxi_utils/util/connect/panos.py @@ -214,7 +214,7 @@ def show_routing_ospf(self) -> typing.Dict[str, typing.Any]: routes.append(value) flags = {} - for m in re.findall('[^\s,]+:[^\s,]+', xml.findtext("./result/flags")): + for m in re.findall(r'[^\s,]+:[^\s,]+', xml.findtext("./result/flags")): field, value = m.split(':') flags[field] = value diff --git a/esxi_utils/vm/virtualmachine.py b/esxi_utils/vm/virtualmachine.py index de0d93a..f33f80e 100644 --- a/esxi_utils/vm/virtualmachine.py +++ b/esxi_utils/vm/virtualmachine.py @@ -35,9 +35,14 @@ class VirtualMachineList: The collection of all virtual machines on the ESXi host. :param client: The `ESXiClient` for the ESXi host. + :param container: The ESXi container object to use when listing VMs + :param legacy_list: When collecting all Virtual Machines from the ESXi server, this setting decides whether the VMs collected are only from the child server on vCenter (True) or if the VM list contains all VMs as seen via the vCenter inventory (False). """ - def __init__(self, client: 'ESXiClient'): + def __init__(self, client: 'ESXiClient', container=None, legacy_list=False): self._client = client + # optional vim.Folder (rootFolder, datacenter.vmFolder, etc.) + self.container = container + self.is_legacy_list = legacy_list def __iter__(self) -> typing.Iterator['VirtualMachine']: from esxi_utils.vm.types.ostype import OSType @@ -146,9 +151,11 @@ def create( folder_name: typing.Optional[str] = None, video_card_auto_detect: typing.Optional[bool] = None, uefi_boot: typing.Optional[bool] = None, + host: typing.Optional[typing.Union[str, pyVmomi.vim.HostSystem]] = None, + resource_pool: typing.Optional[pyVmomi.vim.ResourcePool] = None ) -> 'VirtualMachine': """ - Create a pre-configured VM. + Create a pre-configured VM. The VM will be created on the same host as the 'child' host setting in vCenter. :param name: The name of the new VM. @@ -170,6 +177,10 @@ def create( Will throw an esxi_utils 'MultipleFoldersFoundError' exception if more than one folder is found with the given name. :param uefi_boot: When set to 'True' this VM will be created to emulate secure boot mode (UEFI) instead of BIOS (legacy) boot mode (BIOS is default). + :param host: + The ESXi host server on which to deploy this VM ('child' server in vCenter). When this is 'None', the legacy operation will be performed whereby this VM will be deployed on the same host as the 'child' connected to via this client. + :param resource_pool: + The resource pool to place the VM in. If you want to use the 'host' param pool leave this set to 'None'. :return: A `VirtualMachine` object (or subtype) for the new VM. """ @@ -231,15 +242,31 @@ def create( config.deviceChange.append(video_card) + # Configure destination host and resource pool + if host is None: + target_host = getattr(self._client, "_host_system", None) + elif isinstance(host, pyVmomi.vim.HostSystem): + target_host = host + elif isinstance(host, str): + host_key = host.strip().lower() + matches = [h for h in self._client._all_host_systems if h.name.strip().lower() == host_key] + if not matches: + raise Exception(f"Unable to locate ESXi child host with name: {host}") + if len(matches) > 1: + raise exceptions.MultipleHostSystemsFoundError(self._client._all_host_systems) + target_host = matches[0] + + if resource_pool is None and target_host is not None: + resource_pool = target_host.parent.resourcePool + # Add VM - destination_host = self._client._host_system root_folder = datastore._datacenter.vmFolder folder = VirtualMachineList._get_folder(root_folder, folder_name) vim_vm = self._client._wait_for_task(folder.CreateVM_Task( config, - pool=destination_host.parent.resourcePool, - host=destination_host + pool=resource_pool, + host=target_host )) return self.get(str(vim_vm._moId), search_type='id') @@ -329,9 +356,18 @@ def _get_folder(root_folder, folder_name: str): log.debug(f'VM will upload in folder name: {folder.name}') return folder - def upload(self, file: typing.Union[str, OvfFile], datastore: typing.Union[str, 'Datastore'], name: typing.Optional[str] = None, network_mappings: typing.Optional[typing.Dict[str, str]] = None, folder_name: typing.Optional[str] = None) -> 'VirtualMachine': + def upload( + self, + file: typing.Union[str, OvfFile], + datastore: typing.Union[str, 'Datastore'], + name: typing.Optional[str] = None, + network_mappings: typing.Optional[typing.Dict[str, str]] = None, + folder_name: typing.Optional[str] = None, + host: typing.Optional[typing.Union[str, pyVmomi.vim.HostSystem]] = None, + resource_pool: typing.Optional[pyVmomi.vim.ResourcePool] = None + ) -> 'VirtualMachine': """ - Uploads a local OVF or OVA file to the provided datastore as a new VM. + Uploads a local OVF or OVA file to the provided datastore as a new VM. On vCenter, the VM will be created on the same 'child' host that this client was connected to. :param file: A path to a .ovf/.ova file (string), or an `OvfFile` object. :param datastore: The datastore where the VM should be created. This can be provided as a string (the name of the datastore) or as a `Datastore` object. @@ -341,6 +377,10 @@ def upload(self, file: typing.Union[str, OvfFile], datastore: typing.Union[str, The folder to contain the new VM. The default is the 'root' VMs folder (if the value 'None' is provided) Will create a new folder if one is not found with a matching name. Will throw an esxi_utils 'MultipleFoldersFoundError' exception if more than one folder is found with the given name. + :param host: + The ESXi host server on which to deploy this VM ('child' server in vCenter). When this is 'None', the legacy operation will be performed whereby this VM will be deployed on the same host as the 'child' connected to via this client. + :param resource_pool: + The resource pool to place the VM in. If you want to use the 'host' param pool leave this set to 'None'. :return: A `virtualmachine.VirtualMachine` object for the new VM. """ @@ -387,9 +427,25 @@ def upload(self, file: typing.Union[str, OvfFile], datastore: typing.Union[str, raise exceptions.OvfImportError(file.path, datastore.name, name, str(f"unable to map network {network_name} to {mapped_to}. Reason: {e}")) mappings.append(pyVmomi.vim.OvfManager.NetworkMapping(name=network_name, network=network_obj)) + # Configure destination host and resource pool + if host is None: + target_host = getattr(self._client, "_host_system", None) + elif isinstance(host, pyVmomi.vim.HostSystem): + target_host = host + elif isinstance(host, str): + host_key = host.strip().lower() + matches = [h for h in self._client._all_host_systems if h.name.strip().lower() == host_key] + if not matches: + raise Exception(f"Unable to locate ESXi child host with name: {host}") + if len(matches) > 1: + raise exceptions.MultipleHostSystemsFoundError(self._client._all_host_systems) + target_host = matches[0] + + if resource_pool is None and target_host is not None: + resource_pool = target_host.parent.resourcePool + # Create import spec log.debug(f"Creating import spec...") - resource_pool = self._client._host_system.parent.resourcePool import_spec = self._client._service_instance.content.ovfManager.CreateImportSpec( ovfDescriptor=file.descriptor.xml(pretty_print=True, xml_declaration=True), resourcePool=resource_pool, @@ -446,7 +502,7 @@ def read(self, n=-1): lease = resource_pool.ImportVApp( spec=import_spec.importSpec, folder=folder, - host=self._client._host_system + host=target_host ) # Wait for ready @@ -562,10 +618,37 @@ def _query_vm_properties(self, properties): def _get_vim_vm_objects(self) -> typing.List[typing.Any]: """ Get all pyVmomi VM objects. + Updated 16MAR2026: this now provides a view of all VMs on all child hosts in vCenter. + If you want the previous behavior, collect the list with is_legacy_list set to True. :return: A list of all pyVmomi VM objects. """ - return [ vm for vm in self._client._get_vim_objects(pyVmomi.vim.VirtualMachine) ] + if self.is_legacy_list: + return [ vm for vm in self._client._get_vim_objects(pyVmomi.vim.VirtualMachine) ] + + # The 'new' way is to discover all VMs from the vCenter level if available using 'container' views + content = self._client._content() + + # If caller provided a container, use it + if self.container is not None: + container = self.container + else: + # vCenter: default to inventory-wide search + if self._client.is_vcenter(): + container = content.rootFolder + else: + # standalone ESXi: host-scoped is fine + container = self._client._host_system + + view = content.viewManager.CreateContainerView( + container=container, + type=[pyVmomi.vim.VirtualMachine], + recursive=True + ) + try: + return list(view.view) + finally: + view.Destroy() def _get_guest_id(self, vim_vm) -> typing.Union[str, None]: """ @@ -589,7 +672,9 @@ def _get_guest_id(self, vim_vm) -> typing.Union[str, None]: return None def __str__(self): - return f"<{type(self).__name__} for {self._client.hostname} ({len(self.items)} virtual machines)>" + if self.is_legacy_list: + return f"<{type(self).__name__} for {self._client.hostname} ({len(self.items)} virtual machines)>" + return f"<{type(self).__name__} for inventory ({len(self.items)} virtual machines)>" def __repr__(self): return str(self) @@ -774,6 +859,13 @@ def bootTime(self) -> datetime.datetime: The time the VM was booted """ return self._vim_vm.runtime.bootTime + + @property + def is_template(self) -> bool: + """ + Whether this VM is an inventory 'Template' object in vCenter. + """ + return bool(getattr(self._vim_vm.config, "template", False)) @property def vcpus(self) -> int: @@ -879,6 +971,101 @@ def ostype(self) -> 'OSType': from esxi_utils.vm.types.ostype import OSType return OSType.Unknown + def assert_vcenter(self, err: typing.Optional[str]): + """ + Raises an exception if the VM is not part of a vCenter inventory. + """ + about = self._client._service_instance.content.about + if str(getattr(about, "apiType", "")).lower() != "virtualcenter": + err_msg = err if err else "The requested operation requires a vCenter connection (apiType != VirtualCenter)." + raise Exception(err_msg) + + def to_template(self) -> 'VirtualMachine': + """ + Converts a VM to a template that can be quickly cloned (~two minutes) into new Virtual Machines. + """ + self.assert_vcenter("Template operations require a vCenter connection (apiType != VirtualCenter).") + if self.is_template: + return self + self.assert_powered_off() + try: + self._vim_vm.MarkAsTemplate() + except pyVmomi.vmodl.MethodFault as e: + raise Exception(f"Failed to convert VM '{self.name}' to template: {str(e)}") + return self + + def deploy_from_template( + self, + name: str, + datastore: typing.Union[str, 'Datastore'], + folder_name: typing.Optional[str] = None, + host: typing.Optional[typing.Union[str, pyVmomi.vim.HostSystem]] = None, + resource_pool: typing.Optional[pyVmomi.vim.ResourcePool] = None, + power_on: bool = False, + ) -> 'VirtualMachine': + """ + Attempts to deploy a VM from this template. This VM must already be marked as a template in order for this function to succeed without error (see to_template()) + + :param name: Name of the new VM. + :param datastore: The datastore where the VM should be created. This can be provided as a string (the name of the datastore) or as a `Datastore` object. + :param folder_name: The location in the datastore to place the VM (will be auto-assigned to datastore root if 'None') + :param host: The host to place the new VM on (in vCenter the 'child' ESXi server). + :param resource_pool: The resource pool to place the VM in. If you want to use the 'host' param pool leave this set to 'None'. + :power_on: Whether to turn the VM on or not after the clone (template deploy) operation + """ + self.assert_vcenter("Template operations require a vCenter connection (apiType != VirtualCenter).") + if not self.is_template: + raise TypeError(f"'{self.name}' is not a template (config.template != True).") + + if not isinstance(name, str) or not name.strip(): + raise ValueError("VM 'name' must be a non-empty string") + + if name in self._client.vms.names: + raise exceptions.VirtualMachineExistsError(name) + + if isinstance(datastore, str): + datastore = self._client.datastores.get(datastore) + if not isinstance(datastore, Datastore): + raise TypeError(f"'datastore' is not a valid Datastore object (nor a valid string name for the datastore)") + + root_folder = datastore._datacenter.vmFolder + folder = folder = VirtualMachineList._get_folder(root_folder, folder_name) + + # Configure destination host and resource pool + if host is None: + target_host = getattr(self._client, "_host_system", None) + elif isinstance(host, pyVmomi.vim.HostSystem): + target_host = host + elif isinstance(host, str): + host_key = host.strip().lower() + matches = [h for h in self._client._all_host_systems if h.name.strip().lower() == host_key] + if not matches: + raise Exception(f"Unable to locate ESXi child host with name: {host}") + if len(matches) > 1: + raise exceptions.MultipleHostSystemsFoundError(self._client._all_host_systems) + target_host = matches[0] + + if resource_pool is None and target_host is not None: + resource_pool = target_host.parent.resourcePool + + relocate = pyVmomi.vim.vm.RelocateSpec() + if target_host is not None: + relocate.host = target_host + if resource_pool is not None: + relocate.pool = resource_pool + if datastore is not None: + # pyVmomi expects vim.Datastore here, not your wrapper + relocate.datastore = datastore._vim_datastore if hasattr(datastore, "_vim_datastore") else datastore._datastore # adapt to your wrapper internals + + clone_spec = pyVmomi.vim.vm.CloneSpec( + powerOn=power_on, + template=False, + location=relocate, + ) + + self._client._wait_for_task(self._vim_vm.CloneVM_Task(folder=folder, name=name, spec=clone_spec)) + return self._client.vms.get(name) + def used_space(self, unit: str = "KB") -> int: """ Get the amount of space used on disk by this VM and its associated files. @@ -1440,12 +1627,39 @@ def force_bios_menu(self, enforce:bool = True): @property def _vim_vm(self): + """ + The VM object in pyVmomi + """ vm = pyVmomi.vim.VirtualMachine(self.id) vm._stub = self._client._service_instance._stub return vm + @property + def host_system(self) -> typing.Optional[pyVmomi.vim.HostSystem]: + """ + The ESXi host system this VM is currently running on (or last ran on). + For templates this is usually None. + """ + try: + return getattr(self._vim_vm.runtime, "host", None) or getattr(self._vim_vm.summary.runtime, "host", None) + except pyVmomi.vmodl.fault.ManagedObjectNotFound: + return None + + @property + def esxi_host_name(self) -> typing.Optional[str]: + """ + This is the ESXi host (child in vCenter) in which this VM resides on in the entire inventory. + """ + hs = self.host_system + return hs.name if hs else None + def __str__(self): - return f"<{type(self).__name__} '{self.name}' on {self._client.hostname}>" + host = self.esxi_host_name + # Templates often have no runtime.host; powered-off VMs may show the last host. + if not host: + # Fallback: show the vCenter/selected client host context + host = self._client.hostname + return f"<{type(self).__name__} '{self.name}' on {str(host)}>" def __repr__(self): return str(self) diff --git a/setup.cfg b/setup.cfg index aa12e51..7f7e323 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,6 +1,6 @@ [metadata] name = esxi-utils -version = 3.25.0 +version = 4.0.0 author = The MITRE Corporation description = A package that provides functions for interacting with an ESXi server and manipulating VMs/OVFs. long_description = file: README.md