tests, net, bgp: add connectivity test suite (RedHatQE#2121)

* move udn_vm function to libs

udn_vm is a useful function and
should be placed in a more common location.

Signed-off-by: Sergei Volkov <sevolkov@redhat.com>

* add traffic generator client for pod

For some test suites, such as BGP, it is necessary
to use a traffic generator inside the pod, rather than
in a virtual machine. This commit adds a PodClient class
that represents the tcp client running in the pod.

Also this commit adds "--connect-timeout 300" option to
iperf3 cmd to fix incorrect behavior and prevent false
positives if there is no direct connectivity between
the server and the client.

Signed-off-by: Sergei Volkov <sevolkov@redhat.com>

* add bgp connectivity test suite

Initial BGP testing includes checking the tcp connection between
the CUDN VM and the external network, as well as this connection
preservation during VM migration.

Signed-off-by: Sergei Volkov <sevolkov@redhat.com>

* quarantine tests

while CNV-69734 is open, tests should be quarantined

Signed-off-by: Sergei Volkov <sevolkov@redhat.com>

---------

Signed-off-by: Sergei Volkov <sevolkov@redhat.com>

Author: servolkov

conftest.py

@@ -78,6 +78,7 @@
     "node_remediation",
     "swap",
     "numa",
+    "bgp",
 ]
 
 TEAM_MARKERS = {

libs/net/traffic_generator.py

@@ -1,8 +1,10 @@
 import logging
+from abc import ABC, abstractmethod
 from typing import Final
 
+from ocp_resources.pod import Pod
 from ocp_utilities.exceptions import CommandExecFailed
-from timeout_sampler import TimeoutExpiredError, TimeoutSampler
+from timeout_sampler import TimeoutExpiredError, TimeoutSampler, retry
 
 from libs.vm.vm import BaseVirtualMachine
 
@@ -13,6 +15,27 @@
 LOGGER = logging.getLogger(__name__)
 
 
+class BaseClient(ABC):
+    """Base abstract class for network traffic generator client."""
+
+    def __init__(self, server_ip: str, server_port: int):
+        self._server_ip = server_ip
+        self.server_port = server_port
+        self._cmd = f"{_IPERF_BIN} --client {self._server_ip} --time 0 --port {self.server_port} --connect-timeout 300"
+
+    @abstractmethod
+    def __enter__(self) -> "BaseClient":
+        pass
+
+    @abstractmethod
+    def __exit__(self, exc_type: BaseException, exc_value: BaseException, traceback: object) -> None:
+        pass
+
+    @abstractmethod
+    def is_running(self) -> bool:
+        pass
+
+
 class Server:
     """
     Represents a server running on a virtual machine for testing network performance.
@@ -50,7 +73,7 @@ def is_running(self) -> bool:
         return _is_process_running(vm=self._vm, cmd=self._cmd)
 
 
-class Client:
+class Client(BaseClient):
     """
     Represents a client that connects to a server to test network performance.
     Implemented with iperf3
@@ -67,10 +90,8 @@ def __init__(
         server_ip: str,
         server_port: int,
     ):
+        super().__init__(server_ip=server_ip, server_port=server_port)
         self._vm = vm
-        self._server_ip = server_ip
-        self._server_port = server_port
-        self._cmd = f"{_IPERF_BIN} --client {self._server_ip} --time 0 --port {self._server_port}"
 
     def __enter__(self) -> "Client":
         self._vm.console(
@@ -115,5 +136,47 @@ def _is_process_running(  # type: ignore[return]
         return False
 
 
-def is_tcp_connection(server: Server, client: Client) -> bool:
+class PodClient(BaseClient):
+    """Represents a TCP client that connects to a server to test network performance.
+
+    Expects pod to have iperf3 container.
+
+    Args:
+        pod (Pod): The pod where the client runs.
+        server_ip (str): The destination IP address of the server the client connects to.
+        server_port (int): The port on which the server listens for connections.
+        bind_interface (str): The interface or IP address to bind the client to (optional).
+            If not specified, the client will use the default interface.
+    """
+
+    def __init__(self, pod: Pod, server_ip: str, server_port: int, bind_interface: str | None = None):
+        super().__init__(server_ip=server_ip, server_port=server_port)
+        self._pod = pod
+        self._container = _IPERF_BIN
+        self._cmd += f" --bind {bind_interface}" if bind_interface else ""
+
+    def __enter__(self) -> "PodClient":
+        # run the command in the background using nohup to ensure it keeps running after the exec session ends
+        self._pod.execute(
+            command=["sh", "-c", f"nohup {self._cmd} >/tmp/{_IPERF_BIN}.log 2>&1 &"], container=self._container
+        )
+        self._ensure_is_running()
+
+        return self
+
+    def __exit__(self, exc_type: BaseException, exc_value: BaseException, traceback: object) -> None:
+        self._pod.execute(
+            command=["pkill", "-f", self._cmd],
+        )
+
+    def is_running(self) -> bool:
+        out = self._pod.execute(command=["pgrep", "-f", self._cmd], ignore_rc=True)
+        return bool(out.strip())
+
+    @retry(wait_timeout=30, sleep=2, exceptions_dict={})
+    def _ensure_is_running(self) -> bool:
+        return self.is_running()
+
+
+def is_tcp_connection(server: Server, client: Client | PodClient) -> bool:
     return server.is_running() and client.is_running()

tests/network/bgp/conftest.py

@@ -1,4 +1,5 @@
 import os
+import shlex
 from collections.abc import Generator
 from pathlib import Path
 from typing import Final
@@ -12,10 +13,15 @@
 from ocp_resources.pod import Pod
 
 from libs.net import netattachdef as libnad
+from libs.net.traffic_generator import PodClient as TcpClient
+from libs.net.traffic_generator import Server as TcpServer
 from libs.net.udn import create_udn_namespace
+from libs.net.vmspec import IP_ADDRESS, lookup_iface_status, lookup_primary_network
+from libs.vm.vm import BaseVirtualMachine
 from tests.network.libs import cluster_user_defined_network as libcudn
 from tests.network.libs import nodenetworkconfigurationpolicy as libnncp
 from tests.network.libs.bgp import (
+    POD_SECONDARY_IFACE_NAME,
     create_cudn_route_advertisements,
     create_frr_configuration,
     deploy_external_frr_pod,
@@ -25,13 +31,16 @@
 )
 from tests.network.libs.label_selector import LabelSelector
 from tests.network.libs.nodenetworkstate import DEFAULT_ROUTE_V4, lookup_br_ex_gateway_v4
+from tests.network.libs.vm_factory import udn_vm
 from utilities.infra import get_node_selector_dict
 
 APP_CUDN_LABEL: Final[dict] = {"app": "cudn"}
 BGP_DATA_PATH: Final[Path] = Path(__file__).resolve().parent / "data" / "frr-config"
 CUDN_BGP_LABEL: Final[dict] = {"cudn-bgp": "blue"}
 CUDN_SUBNET_IPV4: Final[str] = "192.168.10.0/24"
-EXTERNAL_SUBNET_IPV4: Final[str] = "172.100.0.0/16"
+EXTERNAL_PROVIDER_SUBNET_IPV4: Final[str] = "10.250.100.0/24"
+EXTERNAL_PROVIDER_IP_V4: Final[str] = "10.250.100.150/24"
+IPERF3_SERVER_PORT: Final[int] = 2354
 
 
 @pytest.fixture(scope="session")
@@ -90,7 +99,7 @@ def frr_configmap(
     workers: list[Node], cnv_tests_utilities_namespace: Namespace, admin_client: DynamicClient
 ) -> Generator[ConfigMap]:
     frr_conf = generate_frr_conf(
-        external_subnet_ipv4=EXTERNAL_SUBNET_IPV4,
+        external_subnet_ipv4=EXTERNAL_PROVIDER_SUBNET_IPV4,
         nodes_ipv4_list=[worker.internal_ip for worker in workers],
     )
 
@@ -156,7 +165,7 @@ def frr_configuration_created(admin_client: DynamicClient) -> Generator[None]:
     with create_frr_configuration(
         name="frr-configuration-bgp",
         frr_pod_ipv4=os.environ["EXTERNAL_FRR_STATIC_IPV4"].split("/")[0],
-        external_subnet_ipv4=EXTERNAL_SUBNET_IPV4,
+        external_subnet_ipv4=EXTERNAL_PROVIDER_SUBNET_IPV4,
         client=admin_client,
     ):
         yield
@@ -179,6 +188,11 @@ def frr_external_pod(
         default_route=br_ex_gateway_v4,
         client=admin_client,
     ) as pod:
+        # Assign a secondary IP on the secondary interface to emulate the external provider subnet
+        pod.execute(
+            command=shlex.split(f"ip addr add {EXTERNAL_PROVIDER_IP_V4} dev {POD_SECONDARY_IFACE_NAME}"),
+            container="frr",
+        )
         yield pod
 
 
@@ -191,3 +205,35 @@ def bgp_setup_ready(
 ) -> None:
     node_names = [worker.name for worker in workers]
     wait_for_bgp_connection_established(node_names=node_names)
+
+
+@pytest.fixture(scope="module")
+def vm_cudn(
+    namespace_cudn: Namespace,
+    cudn_layer2: libcudn.ClusterUserDefinedNetwork,
+) -> Generator[BaseVirtualMachine]:
+    with udn_vm(namespace_name=namespace_cudn.name, name="vm-cudn-bgp") as vm:
+        vm.start(wait=True)
+        vm.wait_for_agent_connected()
+        yield vm
+
+
+@pytest.fixture(scope="module")
+def tcp_server_cudn_vm(vm_cudn: BaseVirtualMachine) -> Generator[TcpServer]:
+    with TcpServer(vm=vm_cudn, port=IPERF3_SERVER_PORT) as server:
+        if not server.is_running():
+            raise ProcessLookupError("Iperf3 server process is not running in the VM")
+        yield server
+
+
+@pytest.fixture(scope="module")
+def tcp_client_external_network(
+    frr_external_pod: Pod, vm_cudn: BaseVirtualMachine, tcp_server_cudn_vm: TcpServer
+) -> Generator[TcpClient]:
+    with TcpClient(
+        pod=frr_external_pod,
+        server_ip=lookup_iface_status(vm=vm_cudn, iface_name=lookup_primary_network(vm=vm_cudn).name)[IP_ADDRESS],
+        server_port=IPERF3_SERVER_PORT,
+        bind_interface=EXTERNAL_PROVIDER_IP_V4.split("/")[0],
+    ) as client:
+        yield client

tests/network/bgp/test_bgp_connectivity.py

@@ -0,0 +1,29 @@
+import pytest
+
+from libs.net.traffic_generator import is_tcp_connection
+from utilities.constants import QUARANTINED
+from utilities.virt import migrate_vm_and_verify
+
+pytestmark = [pytest.mark.bgp, pytest.mark.usefixtures("bgp_setup_ready")]
+
+
+@pytest.mark.polarion("CNV-12276")
+@pytest.mark.xfail(
+    reason=f"{QUARANTINED}: BGP test suite infra dependencies are not met, tracked in CNV-69734",
+    run=False,
+)
+def test_connectivity_cudn_vm_and_external_network(tcp_server_cudn_vm, tcp_client_external_network):
+    assert is_tcp_connection(server=tcp_server_cudn_vm, client=tcp_client_external_network)
+
+
+@pytest.mark.polarion("CNV-12281")
+@pytest.mark.xfail(
+    reason=f"{QUARANTINED}: BGP test suite infra dependencies are not met, tracked in CNV-69734",
+    run=False,
+)
+def test_connectivity_is_preserved_during_cudn_vm_migration(
+    tcp_server_cudn_vm,
+    tcp_client_external_network,
+):
+    migrate_vm_and_verify(vm=tcp_server_cudn_vm.vm)
+    assert is_tcp_connection(server=tcp_server_cudn_vm, client=tcp_client_external_network)

tests/network/bgp/test_bgp_infra.py

@@ -20,8 +20,10 @@
 _CLUSTER_FRR_ASN: Final[int] = 64512
 _EXTERNAL_FRR_ASN: Final[int] = 64000
 _EXTERNAL_FRR_IMAGE: Final[str] = "quay.io/frrouting/frr:9.1.2"
+_IPERF3_IMAGE: Final[str] = "quay.io/networkstatic/iperf3"
 _FRR_DEPLOYMENT_NAME: Final[str] = "frr-k8s-webhook-server"
 _FRR_NS_NAME: Final[str] = "openshift-frr-k8s"
+POD_SECONDARY_IFACE_NAME: Final[str] = "net1"
 
 
 @contextmanager
@@ -184,6 +186,10 @@ def deploy_external_frr_pod(
     attaches it to a specified NetworkAttachmentDefinition (NAD), and mounts a ConfigMap for FRR
     configuration. On exiting the context, the pod is automatically deleted.
 
+    Also contains an iperf3 container to be used for connectivity testing. The process namespace
+    of the iperf3 container is shared with the frr container for the sake of process management
+    (due to the minimal capabilities of the iperf3 container).
+
     Args:
         namespace_name (str): The name of the namespace where the pod will be deployed.
         node_name (str): The name of the node where the pod will be scheduled.
@@ -197,7 +203,7 @@ def deploy_external_frr_pod(
     """
     annotations = {
         f"{Pod.ApiGroup.K8S_V1_CNI_CNCF_IO}/networks": json.dumps([
-            {"name": nad_name, "interface": "net1", "default-route": [default_route]}
+            {"name": nad_name, "interface": POD_SECONDARY_IFACE_NAME, "default-route": [default_route]}
         ]),
         f"{Pod.ApiGroup.K8S_V1_CNI_CNCF_IO}/default-network": "none",
     }
@@ -207,7 +213,12 @@ def deploy_external_frr_pod(
             "image": _EXTERNAL_FRR_IMAGE,
             "securityContext": {"privileged": True, "capabilities": {"add": ["NET_ADMIN"]}},
             "volumeMounts": [{"name": frr_configmap_name, "mountPath": "/etc/frr"}],
-        }
+        },
+        {
+            "name": "iperf3",
+            "image": _IPERF3_IMAGE,
+            "command": ["sleep", "infinity"],
+        },
     ]
     volumes = [{"name": frr_configmap_name, "configMap": {"name": frr_configmap_name}}]
 
@@ -219,6 +230,7 @@ def deploy_external_frr_pod(
         containers=containers,
         volumes=volumes,
         client=client,
+        share_process_namespace=True,
     ) as pod:
         pod.wait_for_status(status=Pod.Status.RUNNING)
         yield pod

tests/network/libs/bgp.py

@@ -0,0 +1,21 @@
+"""This module provides various virtual machine configurations with a focus on network setups."""
+
+from libs.net.udn import udn_primary_network
+from libs.vm.affinity import new_pod_anti_affinity
+from libs.vm.factory import base_vmspec, fedora_vm
+from libs.vm.vm import BaseVirtualMachine
+
+
+def udn_vm(namespace_name: str, name: str, template_labels: dict | None = None) -> BaseVirtualMachine:
+    spec = base_vmspec()
+    iface, network = udn_primary_network(name="udn-primary")
+    spec.template.spec.domain.devices.interfaces = [iface]  # type: ignore
+    spec.template.spec.networks = [network]
+    if template_labels:
+        spec.template.metadata.labels = spec.template.metadata.labels or {}  # type: ignore
+        spec.template.metadata.labels.update(template_labels)  # type: ignore
+        # Use the first label key and first value as the anti-affinity label to use:
+        label, *_ = template_labels.items()
+        spec.template.spec.affinity = new_pod_anti_affinity(label=label)
+
+    return fedora_vm(namespace=namespace_name, name=name, spec=spec)

tests/network/libs/vm_factory.py

@@ -5,11 +5,9 @@
 from ocp_resources.utils.constants import TIMEOUT_1MINUTE
 
 from libs.net.traffic_generator import Client, Server, is_tcp_connection
-from libs.net.udn import udn_primary_network
 from libs.net.vmspec import lookup_iface_status, lookup_primary_network
 from libs.vm import affinity
-from libs.vm.affinity import new_pod_anti_affinity
-from libs.vm.factory import base_vmspec, fedora_vm
+from tests.network.libs.vm_factory import udn_vm
 from utilities.constants import PUBLIC_DNS_SERVER_IP, QUARANTINED, TIMEOUT_1MIN
 from utilities.infra import create_ns
 from utilities.virt import migrate_vm_and_verify
@@ -18,21 +16,6 @@
 SERVER_PORT = 5201
 
 
-def udn_vm(namespace_name, name, template_labels=None):
-    spec = base_vmspec()
-    iface, network = udn_primary_network(name="udn-primary")
-    spec.template.spec.domain.devices.interfaces = [iface]
-    spec.template.spec.networks = [network]
-    if template_labels:
-        spec.template.metadata.labels = spec.template.metadata.labels or {}
-        spec.template.metadata.labels.update(template_labels)
-        # Use the first label key and first value as the anti-affinity label to use:
-        label, *_ = template_labels.items()
-        spec.template.spec.affinity = new_pod_anti_affinity(label=label)
-
-    return fedora_vm(namespace=namespace_name, name=name, spec=spec)
-
-
 @pytest.fixture(scope="module")
 def udn_namespace(admin_client):
     yield from create_ns(