powervs: fix ports cleaning by cleaning them manually on delete

Author: nikromen

pyproject.toml

@@ -27,6 +27,7 @@ resalloc-ibm-cloud-vm = "resalloc_ibm_cloud.ibm_cloud_vm:main"
 resalloc-ibm-cloud-powervs-vm = "resalloc_ibm_cloud.powervs.powervs_vm:main"
 resalloc-ibm-cloud-powervs-list-vms = "resalloc_ibm_cloud.powervs.powervs_list_vms:main"
 resalloc-ibm-cloud-powervs-list-deleting-vms = "resalloc_ibm_cloud.powervs.powervs_list_deleting_vms:main"
+resalloc-ibm-cloud-powervs-cleanup-ips = "resalloc_ibm_cloud.powervs.powervs_cleanup_ips:main"
 
 
 [build-system]
@@ -47,4 +48,5 @@ manpages = [
     "man/resalloc-ibm-cloud-powervs-vm.1:function=powervs_arg_parser:pyfile=resalloc_ibm_cloud/argparsers.py",
     "man/resalloc-ibm-cloud-powervs-list-vms.1:function=powervs_list_vms_parser:pyfile=resalloc_ibm_cloud/argparsers.py",
     "man/resalloc-ibm-cloud-powervs-list-deleting-vms.1:function=powervs_list_deleting_vms_parser:pyfile=resalloc_ibm_cloud/argparsers.py",
+    "man/resalloc-ibm-cloud-powervs-cleanup-ips.1:function=powervs_cleanup_ips_parser:pyfile=resalloc_ibm_cloud/argparsers.py",
 ]

resalloc-ibm-cloud.spec

@@ -52,6 +52,7 @@ BuildRequires:  pyproject-rpm-macros
 %{_bindir}/resalloc-ibm-cloud-powervs-list-deleting-vms
 %{_bindir}/resalloc-ibm-cloud-powervs-list-vms
 %{_bindir}/resalloc-ibm-cloud-powervs-vm
+%{_bindir}/resalloc-ibm-cloud-powervs-cleanup-ips
 
 
 %changelog

resalloc_ibm_cloud/argparsers.py

@@ -247,3 +247,18 @@ def powervs_list_deleting_vms_parser():
     """
     parser = _default_arg_parser_powervs(prog=_pfx("powervs-list-deleting-vms"))
     return parser
+
+
+def powervs_cleanup_ips_parser():
+    """
+    Parser for the resalloc-ibm-cloud-powervs-cleanup-ips utility.
+    """
+    parser = _default_arg_parser_powervs(prog=_pfx("powervs-cleanup-ips"))
+    parser.add_argument(
+        "--network-id",
+        type=str,
+        nargs="+",
+        required=True,
+        help="One or more network (subnet) IDs to clean up orphaned IPs from",
+    )
+    return parser

resalloc_ibm_cloud/powervs/client.py

@@ -63,6 +63,7 @@ def request(
         params: dict = None,
         json_data: dict = None,
         broker: bool = False,
+        v1_api: bool = False,
     ) -> dict:
         """
         Make a request to the PowerVS API with automatic retry for server errors
@@ -78,6 +79,8 @@ def request(
             params: Query parameters
             json_data: JSON body data
             broker: Whether to use the broker API
+            v1_api: Whether to use the newer /v1 API (endpoints migrated
+                from /pcloud/v1); these don't use the cloud-instance prefix
 
         Returns:
             Response JSON
@@ -88,12 +91,15 @@ def request(
             requests.RequestException: For other request-related errors after
                 retries are exhausted
         """
-        base_url = (
-            self.credentials.broker_url if broker else self.credentials.service_url
-        )
-
-        # set prefix path for powervs API workspace
-        if not broker and not path.startswith(
+        if v1_api:
+            base_url = self.credentials.v1_url
+        elif broker:
+            base_url = self.credentials.broker_url
+        else:
+            base_url = self.credentials.service_url
+
+        # set prefix path for powervs API workspace (legacy /pcloud/v1 only)
+        if not broker and not v1_api and not path.startswith(
             f"/cloud-instances/{self.cloud_instance_id}"
         ):
             path = f"/cloud-instances/{self.cloud_instance_id}{path}"
@@ -264,3 +270,40 @@ def list_volumes(self) -> list[dict]:
             List of volumes
         """
         return self.request("GET", "/volumes").get("volumes", [])
+
+    def list_network_interfaces(self, network_id: str) -> list[dict]:
+        """
+        List all network interfaces (ports) on a given network.
+
+        Args:
+            network_id: Network (subnet) ID
+
+        Returns:
+            List of network interfaces
+        """
+        response = self.request(
+            "GET",
+            f"/networks/{network_id}/network-interfaces",
+            v1_api=True,
+        )
+        return response.get("interfaces", [])
+
+    def delete_network_interface(
+        self, network_id: str, interface_id: str
+    ) -> None:
+        """
+        Delete a network interface (port) from a network.
+
+        Args:
+            network_id: Network (subnet) ID
+            interface_id: Network interface ID to delete
+        """
+        logger.info(
+            "Deleting network interface %s from network %s",
+            interface_id, network_id,
+        )
+        self.request(
+            "DELETE",
+            f"/networks/{network_id}/network-interfaces/{interface_id}",
+            v1_api=True,
+        )

resalloc_ibm_cloud/powervs/credentials.py

@@ -32,9 +32,14 @@ def iaas_url(self) -> str:
 
     @property
     def service_url(self) -> str:
-        """PowerVS service URL for the region"""
+        """PowerVS service URL for the region (/pcloud/v1 API path)"""
         return f"{self.iaas_url}/pcloud/v1"
 
+    @property
+    def v1_url(self) -> str:
+        """PowerVS v1 API URL (/v1 - totally different API - no pcloud prefix)"""
+        return f"{self.iaas_url}/v1"
+
     @property
     def broker_url(self) -> str:
         """PowerVS broker URL"""

resalloc_ibm_cloud/powervs/powervs_cleanup_ips.py

@@ -0,0 +1,68 @@
+"""
+Helper script for cleaning up orphaned network interfaces (ports) from
+PowerVS subnets.
+"""
+
+import logging
+
+from resalloc_ibm_cloud.argparsers import powervs_cleanup_ips_parser
+from resalloc_ibm_cloud.helpers import setup_logging
+from resalloc_ibm_cloud.powervs.credentials import get_powervs_credentials
+from resalloc_ibm_cloud.powervs.client import PowerVSClient
+
+
+logger = logging.getLogger(__name__)
+
+
+def _get_active_instance_ids(client: PowerVSClient) -> set[str]:
+    instances = client.list_instances()
+    return {inst["pvmInstanceID"] for inst in instances if inst.get("pvmInstanceID")}
+
+
+def cleanup_network_interfaces(
+    client: PowerVSClient,
+    network_ids: list[str],
+) -> None:
+    """
+    Delete orphaned network interfaces (ports) that are not attached to any
+    active VM instance.
+    """
+    active_ids = _get_active_instance_ids(client)
+
+    logger.info("Found %d active instance(s)", len(active_ids))
+
+    for network_id in network_ids:
+        interfaces = client.list_network_interfaces(network_id)
+        logger.info(
+            "Network %s: found %d network interface(s)",
+            network_id, len(interfaces),
+        )
+
+        orphaned = 0
+        for iface in interfaces:
+            instance_ref = iface.get("instance")
+            instance_id = instance_ref.get("instanceID") if instance_ref else None
+            if instance_id and instance_id in active_ids:
+                continue
+
+            orphaned += 1
+            logger.info(
+                "Deleting orphaned network interface %s (ip=%s) from network %s",
+                iface["id"], iface.get("ipAddress", "?"), network_id,
+            )
+
+            client.delete_network_interface(network_id, iface["id"])
+
+        logger.info(
+            "Network %s: deleted %d orphaned interface(s)", network_id, orphaned,
+        )
+
+
+def main() -> None:
+    """Entrypoint to the script."""
+    opts = powervs_cleanup_ips_parser().parse_args()
+    setup_logging(opts.log_level)
+
+    credentials = get_powervs_credentials(opts.token_file, opts.crn)
+    client = PowerVSClient(credentials)
+    cleanup_network_interfaces(client, opts.network_id)

resalloc_ibm_cloud/powervs/powervs_vm.py

@@ -10,7 +10,7 @@
 from typing import Any, Optional
 
 import backoff
-from requests import HTTPError
+from requests import HTTPError, RequestException
 import requests
 
 from resalloc_ibm_cloud.argparsers import powervs_arg_parser
@@ -162,6 +162,39 @@ def _delete_volume_with_backoff(self, volume_id: str) -> None:
         self.client.delete_volume(volume_id)
         logger.info("Deleted volume with ID %s", volume_id)
 
+    def _wait_for_instance_gone(
+        self, instance_id: str, timeout: int = 600, interval: int = 10,
+    ) -> None:
+        """
+        Poll until the instance no longer exists (404) so that its network
+        interfaces are safe to delete.
+        """
+        start_time = time.time()
+        while True:
+            if time.time() - start_time > timeout:
+                logger.warning(
+                    "Timed out waiting for instance %s to be fully deleted",
+                    instance_id,
+                )
+                break
+
+            try:
+                instance = self.client.get_instance(instance_id)
+                status = instance.get("status", "unknown")
+                logger.info(
+                    "Waiting for instance %s to be deleted (status: %s)",
+                    instance_id, status,
+                )
+            except RequestException as e:
+                if e.response.status_code == 404:
+                    logger.info("Instance %s is gone", instance_id)
+                    return
+
+                logger.error("Failed to get instance %s: %s", instance_id, str(e))
+                break
+
+            sleep(interval)
+
     def _force_delete_volume_by_instance_name(self, instance_name: str) -> None:
         # if powervs decides to fail and keep the volume around, force delete any
         # volume that starts with the instance name
@@ -198,6 +231,18 @@ def delete_vm(self, name: str) -> None:
         instance_information = self.client.get_instance(instance_id)
         volume_ids = instance_information.get("volumeIDs", [])
 
+        # IBM Cloud does not automatically clean up network interfaces (ports)
+        # when a VM is deleted, which eventually exhausts the subnet IP pool.
+        # We need to remember them now and delete them after the instance is gone.
+        # This is a new thing in PowerVS, maybe in the future it will be handled automatically
+        # again???
+        network_interfaces = []
+        for net in instance_information.get("networks", []):
+            network_id = net.get("networkID")
+            interface_id = net.get("networkInterfaceID")
+            if network_id and interface_id:
+                network_interfaces.append((network_id, interface_id))
+
         # the data volumes tends to remain undeleted even if the delete_instance
         # call is with delete_data_volumes, so this needs to be assured manually
         for volume_id in volume_ids:
@@ -218,6 +263,19 @@ def delete_vm(self, name: str) -> None:
             instance_id,
         )
 
+        if not network_interfaces:
+            return
+
+        self._wait_for_instance_gone(instance_id)
+        for network_id, interface_id in network_interfaces:
+            try:
+                self.client.delete_network_interface(network_id, interface_id)
+            except RequestException as e:
+                logger.error(
+                    "Failed to delete network interface %s: %s",
+                    interface_id, str(e),
+                )
+
     def _parse_volumes(self, volumes_list: list[str]) -> list[dict]:
         """
         Parse volume specifications from a list of strings.