net, tests, stuntime: Add OVN localnet migration stuntime scenario

Implement the initial stuntime scenario to serve as a baseline for future
performance testing.

For now, the global stuntime threshold is set to a 5s placeholder. Once
we finish automating the remaining scenarios and have the baseline data
to calibrate our expectations, we’ll replace this with a more precise,
data-driven value.

Technical changes:
 - Annotate get_node_selector_dict to satisfy strict mypy checks
   (disallow_untyped_calls) for the new localnet migration fixture.
 - Stuntime scenarios require two VMs on the same node. To support this,
   anti-affinity is now configurable; it remains enabled by default to
   avoid impacting existing callers.

Signed-off-by: Anat Wax <awax@redhat.com>
Assisted by: Cursor

Author: Anatw

libs/vm/spec.py

@@ -31,6 +31,7 @@ class VMISpec:
     volumes: list[Volume] | None = None
     terminationGracePeriodSeconds: int | None = None  # noqa: N815
     affinity: Affinity | None = None
+    nodeSelector: dict[str, str] | None = None  # noqa: N815
 
 
 @dataclass

libs/vm/vm.py

@@ -114,6 +114,16 @@ def update_template_annotations(self, template_annotations: dict[str, str]) -> N
         }
         ResourceEditor(patches=patches).update()
 
+    def update_template_node_selector(self, node_selector: dict[str, str] | None) -> None:
+        """Update the VM template node selector.
+
+        Args:
+            node_selector: Node selector dictionary to apply to the VM template spec.
+                Set to None to clear the node selector.
+        """
+        patches = {self: {"spec": {"template": {"spec": {"nodeSelector": node_selector}}}}}
+        ResourceEditor(patches=patches).update()
+
     @property
     def template_spec(self) -> VMISpec:
         return self._spec.template.spec

tests/network/localnet/liblocalnet.py

@@ -1,7 +1,8 @@
 import contextlib
 import logging
 import uuid
-from typing import Final, Generator
+from collections.abc import Generator
+from typing import Final
 
 from kubernetes.client import ApiException
 from kubernetes.dynamic import DynamicClient
@@ -77,6 +78,8 @@ def localnet_vm(
     networks: list[Network],
     interfaces: list[Interface],
     network_data: cloudinit.NetworkData | None = None,
+    pod_anti_affinity: bool = True,
+    node_selector: dict[str, str] | None = None,
 ) -> BaseVirtualMachine:
     """
     Create a Fedora-based Virtual Machine connected to localnet network(s).
@@ -95,6 +98,8 @@ def localnet_vm(
             Each Interface should have a name matching a Network, and additional configuration and state.
         network_data (cloudinit.NetworkData | None): Cloud-init NetworkData object containing the network
             configuration for the VM interfaces. If None, no network configuration is applied via cloud-init.
+        pod_anti_affinity (bool): When True (default), apply localnet pod anti-affinity like other localnet VMs.
+        node_selector (dict[str, str] | None): Optional VMI nodeSelector (e.g. pin to a worker hostname).
 
     Returns:
         BaseVirtualMachine: The configured VM object ready for creation.
@@ -136,8 +141,14 @@ def localnet_vm(
         )
         vmi_spec = add_volume_disk(vmi_spec=vmi_spec, volume=volume, disk=disk)
 
-    vmi_spec.affinity = new_pod_anti_affinity(label=next(iter(LOCALNET_TEST_LABEL.items())))
-    vmi_spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].namespaceSelector = {}
+    if pod_anti_affinity:
+        vmi_spec.affinity = new_pod_anti_affinity(label=next(iter(LOCALNET_TEST_LABEL.items())))
+        vmi_spec.affinity.podAntiAffinity.requiredDuringSchedulingIgnoredDuringExecution[0].namespaceSelector = {}
+    else:
+        vmi_spec.affinity = None
+
+    if node_selector is not None:
+        vmi_spec.nodeSelector = node_selector
 
     return fedora_vm(namespace=namespace, name=name, client=client, spec=spec)
 

tests/network/localnet/migration_stuntime/conftest.py

@@ -0,0 +1,88 @@
+import logging
+from collections.abc import Generator
+
+import pytest
+from kubernetes.dynamic import DynamicClient
+from ocp_resources.namespace import Namespace
+
+from libs.vm.spec import Interface, Multus, Network
+from libs.vm.vm import BaseVirtualMachine
+from tests.network.libs import cloudinit
+from tests.network.libs import cluster_user_defined_network as libcudn
+from tests.network.localnet.liblocalnet import (
+    GUEST_1ST_IFACE_NAME,
+    LOCALNET_OVS_BRIDGE_INTERFACE,
+    ip_addresses_from_pool,
+    libnncp,
+    localnet_vm,
+    run_vms,
+)
+from utilities.infra import get_node_selector_dict
+
+LOGGER = logging.getLogger(__name__)
+
+
+def _create_localnet_stuntime_vm(
+    namespace: str,
+    name: str,
+    client: DynamicClient,
+    cudn_name: str,
+    ipv4_pool: Generator[str],
+    ipv6_pool: Generator[str],
+    node_selector: dict[str, str] | None = None,
+) -> BaseVirtualMachine:
+    """Create a localnet VM for stuntime testing with optional node selector."""
+    return localnet_vm(
+        namespace=namespace,
+        name=name,
+        client=client,
+        networks=[Network(name=LOCALNET_OVS_BRIDGE_INTERFACE, multus=Multus(networkName=cudn_name))],
+        interfaces=[Interface(name=LOCALNET_OVS_BRIDGE_INTERFACE, bridge={})],
+        network_data=cloudinit.NetworkData(
+            ethernets={
+                GUEST_1ST_IFACE_NAME: cloudinit.EthernetDevice(
+                    addresses=ip_addresses_from_pool(
+                        ipv4_pool=ipv4_pool,
+                        ipv6_pool=ipv6_pool,
+                    )
+                )
+            }
+        ),
+        pod_anti_affinity=False,
+        node_selector=node_selector,
+    )
+
+
+@pytest.fixture()
+def ovs_bridge_localnet_same_node_running_server_client(
+    unprivileged_client: DynamicClient,
+    nncp_localnet_on_secondary_node_nic: libnncp.NodeNetworkConfigurationPolicy,
+    cudn_localnet_ovs_bridge: libcudn.ClusterUserDefinedNetwork,
+    namespace_localnet_1: Namespace,
+    ipv4_localnet_address_pool: Generator[str],
+    ipv6_localnet_address_pool: Generator[str],
+) -> Generator[tuple[BaseVirtualMachine, BaseVirtualMachine]]:
+    """
+    Server and client Fedora VMs on OVS localnet, both scheduled on the same worker node.
+    """
+    with _create_localnet_stuntime_vm(
+        namespace=namespace_localnet_1.name,
+        name="localnet-stuntime-server",
+        client=unprivileged_client,
+        cudn_name=cudn_localnet_ovs_bridge.name,
+        ipv4_pool=ipv4_localnet_address_pool,
+        ipv6_pool=ipv6_localnet_address_pool,
+    ) as server_vm:
+        run_vms(vms=(server_vm,))
+        server_node_name = server_vm.vmi.node.name
+        with _create_localnet_stuntime_vm(
+            namespace=namespace_localnet_1.name,
+            name="localnet-stuntime-client",
+            client=unprivileged_client,
+            cudn_name=cudn_localnet_ovs_bridge.name,
+            ipv4_pool=ipv4_localnet_address_pool,
+            ipv6_pool=ipv6_localnet_address_pool,
+            node_selector=get_node_selector_dict(node_selector=server_node_name),
+        ) as client_vm:
+            run_vms(vms=(client_vm,))
+            yield server_vm, client_vm

tests/network/localnet/migration_stuntime/lib_helpers.py

@@ -0,0 +1,121 @@
+"""Helpers for OVN localnet migration stuntime tests."""
+
+from __future__ import annotations
+
+import ipaddress
+import logging
+import re
+from typing import Final
+
+from libs.vm.vm import BaseVirtualMachine
+
+LOGGER = logging.getLogger(__name__)
+
+GLOBAL_STUNTIME_THRESHOLD_SECONDS: Final[float] = 5.0
+STUNTIME_PING_LOG_PATH: Final[str] = "/tmp/stuntime-ping.log"
+PING_INTERVAL_SECONDS: Final[float] = 0.1
+
+
+class InsufficientStuntimeDataError(ValueError):
+    """Raised when ping log has too few successful replies to compute stuntime."""
+
+
+def _get_ping_ipv6_flag(destination_ip: str) -> str:
+    """Return the ping IPv6 flag based on the IP address version."""
+    ip = ipaddress.ip_address(address=destination_ip)
+    return " -6" if ip.version == 6 else ""
+
+
+def compute_stuntime(ping_log: str) -> float:
+    """Parse ping summary output and compute stuntime from lost packets.
+
+    Uses the summary line `<tx> packets transmitted, <rx> received` and converts
+    packet loss into stuntime using ping interval.
+
+    Args:
+        ping_log: Tail output from ping including summary lines.
+
+    Returns:
+        float: Stuntime in seconds.
+
+    Raises:
+        InsufficientStuntimeDataError: When summary line with transmitted/received is missing.
+    """
+    summary_match = re.search(r"(\d+)\s+packets transmitted,\s+(\d+)\s+received", ping_log)
+    if not summary_match:
+        raise InsufficientStuntimeDataError("Insufficient data to compute stuntime (missing ping summary line)")
+
+    transmitted_packets = int(summary_match.group(1))
+    received_packets = int(summary_match.group(2))
+    lost_packets = transmitted_packets - received_packets
+    stuntime = lost_packets * PING_INTERVAL_SECONDS
+    LOGGER.info(f"Ping lost={lost_packets}, stuntime={stuntime:.1f}s")
+    return stuntime
+
+
+def clear_vm_template_node_selector(vm: BaseVirtualMachine) -> None:
+    """Clear the VM template node selector to allow migration to any node.
+
+    Args:
+        vm: The virtual machine to update.
+    """
+    vm.update_template_node_selector(node_selector=None)
+
+
+def verify_ping_reaches_destination(
+    source_vm: BaseVirtualMachine,
+    destination_ip: str,
+) -> None:
+    """Verify network connectivity from source VM to destination IP.
+
+    Args:
+        source_vm: The virtual machine from which to initiate the ping.
+        destination_ip: The target IP address (IPv4 or IPv6) to ping.
+    """
+    ping_ipv6_flag = _get_ping_ipv6_flag(destination_ip=destination_ip)
+    source_vm.console(
+        commands=[f"ping{ping_ipv6_flag} -q -c 3 {destination_ip}"],
+        timeout=30,
+    )
+
+
+def start_continuous_ping(
+    source_vm: BaseVirtualMachine,
+    destination_ip: str,
+) -> None:
+    """Start a continuous background ping for stuntime measurement.
+
+    Args:
+        source_vm: The virtual machine from which to initiate the continuous ping.
+        destination_ip: The target IP address (IPv4 or IPv6) to ping continuously.
+    """
+    verify_ping_reaches_destination(source_vm=source_vm, destination_ip=destination_ip)
+    ping_ipv6_flag = _get_ping_ipv6_flag(destination_ip=destination_ip)
+    source_vm.console(
+        commands=[
+            f"ping{ping_ipv6_flag} -O -i {PING_INTERVAL_SECONDS} {destination_ip} >{STUNTIME_PING_LOG_PATH} 2>&1 &",
+        ],
+        timeout=10,
+    )
+    LOGGER.info(f"Started timestamped ping from {source_vm.name} to {destination_ip} (log {STUNTIME_PING_LOG_PATH})")
+
+
+def stop_continuous_ping_and_read_log(ping_source_vm: BaseVirtualMachine) -> str:
+    """Stop the continuous ping process and retrieve the summary statistics.
+
+    Args:
+        ping_source_vm: The virtual machine running the continuous ping.
+
+    Returns:
+        str: Ping summary containing packet transmission statistics.
+    """
+    cmd_pkill = "sudo sh -c 'pkill -SIGINT -x ping || true'"
+    cmd_tail = f"sudo tail -n 3 {STUNTIME_PING_LOG_PATH}"
+    lines_by_cmd = ping_source_vm.console(
+        commands=[cmd_pkill, cmd_tail],
+        timeout=120,
+    )
+    assert lines_by_cmd is not None, "Failed to stop continuous ping and read log"
+    ping_log = "\n".join(lines_by_cmd[cmd_tail])
+    LOGGER.info(f"Collected ping summary tail from {ping_source_vm.name}")
+    return ping_log

tests/network/localnet/migration_stuntime/test_migration_stuntime.py

@@ -20,14 +20,23 @@
 
 import pytest
 
-__test__ = False
+from libs.net.vmspec import lookup_iface_status_ip
+from tests.network.localnet.liblocalnet import LOCALNET_OVS_BRIDGE_INTERFACE
+from tests.network.localnet.migration_stuntime.lib_helpers import (
+    GLOBAL_STUNTIME_THRESHOLD_SECONDS,
+    clear_vm_template_node_selector,
+    compute_stuntime,
+    start_continuous_ping,
+    stop_continuous_ping_and_read_log,
+)
+from utilities.virt import migrate_vm_and_verify
+
+MIGRATION_STUNTIME_IP_FAMILY_PARAMS = (
+    pytest.param(4, id="ipv4", marks=pytest.mark.ipv4),
+    pytest.param(6, id="ipv6", marks=pytest.mark.ipv6),
+)
 
 """
-Parametrize:
-    - ip_family:
-        - ipv4 [Markers: ipv4]
-        - ipv6 [Markers: ipv6]
-
 Preconditions:
     - Shared under-test server VM on OVN localnet secondary network, for the IP family from ip_family parametrization.
     - Shared under-test client VM on OVN localnet secondary network, for that same IP family,
@@ -37,8 +46,13 @@
 
 @pytest.mark.incremental
 class TestMigrationStuntime:
+    @pytest.mark.parametrize("ip_family", MIGRATION_STUNTIME_IP_FAMILY_PARAMS)
     @pytest.mark.polarion("CNV-15258")
-    def test_client_migrates_off_server_node(self):
+    def test_client_migrates_off_server_node(
+        self,
+        ovs_bridge_localnet_same_node_running_server_client,
+        ip_family: int,
+    ):
         """
         Test that measured stuntime does not exceed the global threshold when the client
         VM migrates from the node hosting the server VM into a different node.
@@ -58,6 +72,24 @@ def test_client_migrates_off_server_node(self):
         Expected:
             - Measured stuntime does not exceed the global threshold.
         """
+        server_vm, client_vm = ovs_bridge_localnet_same_node_running_server_client
+        server_ip = str(
+            lookup_iface_status_ip(
+                vm=server_vm,
+                iface_name=LOCALNET_OVS_BRIDGE_INTERFACE,
+                ip_family=ip_family,
+            )
+        )
+        clear_vm_template_node_selector(vm=client_vm)
+        start_continuous_ping(
+            source_vm=client_vm,
+            destination_ip=server_ip,
+        )
+        migrate_vm_and_verify(vm=client_vm)
+        measured = compute_stuntime(ping_log=stop_continuous_ping_and_read_log(ping_source_vm=client_vm))
+        assert measured <= GLOBAL_STUNTIME_THRESHOLD_SECONDS, (
+            f"IPv{ip_family} stuntime {measured}s exceeds global threshold ({GLOBAL_STUNTIME_THRESHOLD_SECONDS}s)"
+        )
 
     @pytest.mark.polarion("CNV-15259")
     def test_client_migrates_between_non_server_nodes(self):
@@ -168,3 +200,9 @@ def test_server_migrates_to_client_node(self):
         Expected:
             - Measured stuntime does not exceed the global threshold.
         """
+
+    test_client_migrates_between_non_server_nodes.__test__ = False
+    test_client_migrates_to_server_node.__test__ = False
+    test_server_migrates_off_client_node.__test__ = False
+    test_server_migrates_between_non_client_nodes.__test__ = False
+    test_server_migrates_to_client_node.__test__ = False

utilities/infra.py

@@ -1142,7 +1142,7 @@ def get_node_selector_name(node_selector):
     return node_selector[f"{Resource.ApiGroup.KUBERNETES_IO}/hostname"]
 
 
-def get_node_selector_dict(node_selector):
+def get_node_selector_dict(node_selector: str) -> dict[str, str]:
     return {f"{Resource.ApiGroup.KUBERNETES_IO}/hostname": node_selector}