Skip to content
18 changes: 16 additions & 2 deletions tests/observability/metrics/conftest.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
get_vm_comparison_info_dict,
Comment thread
OhadRevah marked this conversation as resolved.
get_vmi_guest_os_kernel_release_info_metric_from_vm,
metric_result_output_dict_by_mountpoint,
network_packets_received,
vnic_info_from_vm_or_vmi,
)
from tests.observability.utils import validate_metrics_value
Expand Down Expand Up @@ -233,17 +234,30 @@ def generated_network_traffic_windows_vm(windows_vm_for_test):
src_vm=windows_vm_for_test,
dst_ip=get_ip_from_vm_or_virt_handler_pod(family=IPV4_STR, vm=windows_vm_for_test),
windows=True,
quiet_output=False,
)


@pytest.fixture(scope="class")
def linux_vm_for_test_interface_name(vm_for_test):
return vm_for_test.vmi.interfaces[0].interfaceName
return vm_for_test.vmi.interfaces[0].podInterfaceName


@pytest.fixture(scope="class")
def windows_vm_for_test_interface_name(windows_vm_for_test):
return windows_vm_for_test.vmi.interfaces[0].interfaceName
return windows_vm_for_test.vmi.interfaces[0].podInterfaceName


@pytest.fixture()
def network_packet_received_windows_vm(windows_vm_for_test, windows_vm_for_test_interface_name):
return network_packets_received(
Comment thread
OhadRevah marked this conversation as resolved.
vm=windows_vm_for_test, interface_name=windows_vm_for_test_interface_name, windows_wsl=True
)


@pytest.fixture()
def network_packet_received_linux_vm(vm_for_test, linux_vm_for_test_interface_name):
return network_packets_received(vm=vm_for_test, interface_name=linux_vm_for_test_interface_name)
Comment thread
OhadRevah marked this conversation as resolved.


@pytest.fixture(scope="class")
Expand Down
20 changes: 14 additions & 6 deletions tests/observability/metrics/test_network_metrics.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,23 +17,31 @@ class TestVmiNetworkMetricsLinux:
@pytest.mark.polarion("CNV-11177")
@pytest.mark.s390x
def test_kubevirt_vmi_network_traffic_bytes_total(
self, prometheus, vm_for_test, linux_vm_for_test_interface_name, generated_network_traffic
self,
prometheus,
vm_for_test,
linux_vm_for_test_interface_name,
generated_network_traffic,
network_packet_received_linux_vm,
):
validate_network_traffic_metrics_value(
prometheus=prometheus,
vm=vm_for_test,
interface_name=linux_vm_for_test_interface_name,
network_packet_received=network_packet_received_linux_vm,
)


@pytest.mark.tier3
class TestVmiNetworkMetricsWindows:
@pytest.mark.polarion("CNV-11846")
def test_kubevirt_vmi_network_traffic_bytes_total_windows_vm(
self, prometheus, windows_vm_for_test, windows_vm_for_test_interface_name, generated_network_traffic_windows_vm
self,
prometheus,
windows_vm_for_test,
windows_vm_for_test_interface_name,
generated_network_traffic_windows_vm,
network_packet_received_windows_vm,
):
Comment thread
OhadRevah marked this conversation as resolved.
validate_network_traffic_metrics_value(
prometheus=prometheus,
vm=windows_vm_for_test,
interface_name=windows_vm_for_test_interface_name,
prometheus=prometheus, vm=windows_vm_for_test, network_packet_received=network_packet_received_windows_vm
)
77 changes: 38 additions & 39 deletions tests/observability/metrics/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@
import math
import re
import shlex
import time
import urllib
from contextlib import contextmanager
from datetime import datetime, timezone
Expand Down Expand Up @@ -31,12 +30,13 @@
cleanup_artifactory_secret_and_config_map,
get_artifactory_config_map,
get_artifactory_secret,
get_http_image_url,
get_test_artifact_server_url,
)
from utilities.constants import (
CAPACITY,
NODE_STR,
OS_FLAVOR_WINDOWS,
REGISTRY_STR,
TIMEOUT_1MIN,
TIMEOUT_2MIN,
TIMEOUT_4MIN,
Expand All @@ -45,6 +45,7 @@
TIMEOUT_15SEC,
TIMEOUT_20SEC,
TIMEOUT_30SEC,
TIMEOUT_40MIN,
USED,
VIRT_HANDLER,
Images,
Expand Down Expand Up @@ -323,37 +324,32 @@ def validate_metric_value_within_range(
raise


def network_packets_received(vm: VirtualMachineForTests, interface_name: str) -> dict[str, str]:
ip_link_show_content = run_ssh_commands(host=vm.ssh_exec, commands=shlex.split("ip -s link show"))[0]
def network_packets_received(
vm: VirtualMachineForTests, interface_name: str, windows_wsl: bool = False
) -> dict[str, str]:
ip_link_show_content = run_ssh_commands(
host=vm.ssh_exec, commands=shlex.split(f"{'wsl' if windows_wsl else ''} ip -s link show")
)[0]

pattern = re.compile(
rf".*?{re.escape(interface_name)}:.*?" # Match the line with the interface name
r"(?:RX:\s+bytes\s+packets\s+errors\s+dropped\s+.*?(\d+)\s+(\d+)\s+(\d+)\s+(\d+)).*?" # Capture RX stats
r"(?:TX:\s+bytes\s+packets\s+errors\s+dropped\s+.*?(\d+)\s+(\d+)\s+(\d+)\s+(\d+))", # Capture TX stats
r"RX:.*?\n\s+(?P<rx_bytes>\d+)\s+(?P<rx_packets>\d+)\s+(?P<rx_errs>\d+)\s+(?P<rx_drop>\d+)\s+\d+\s+\d+.*?"
r"TX:.*?\n\s+(?P<tx_bytes>\d+)\s+(?P<tx_packets>\d+)\s+(?P<tx_errs>\d+)\s+(?P<tx_drop>\d+)",
re.DOTALL | re.IGNORECASE,
)
match = pattern.search(string=ip_link_show_content)

if match:
rx_bytes, rx_packets, rx_errs, rx_drop, tx_bytes, tx_packets, tx_errs, tx_drop = match.groups()
return {
"rx_bytes": rx_bytes,
"rx_packets": rx_packets,
"rx_errs": rx_errs,
"rx_drop": rx_drop,
"tx_bytes": tx_bytes,
"tx_packets": tx_packets,
"tx_errs": tx_errs,
"tx_drop": tx_drop,
}
return {}
result = match.groupdict()
LOGGER.info(f"Successfully parsed network stats: {result}")
return result
raise ValueError(f"No match found for interface '{interface_name}' in ip link show output : {ip_link_show_content}")


def compare_network_traffic_bytes_and_metrics(
prometheus: Prometheus, vm: VirtualMachineForTests, vm_interface_name: str
prometheus: Prometheus, vm: VirtualMachineForTests, network_packet_received: dict[str, str]
) -> bool:
packet_received = network_packets_received(vm=vm, interface_name=vm_interface_name)
rx_tx_indicator = False
LOGGER.info("Waiting for metric kubevirt_vmi_network_traffic_bytes_total to update")
time.sleep(TIMEOUT_15SEC)
metric_result = (
prometheus
.query(query=f"kubevirt_vmi_network_traffic_bytes_total{{name='{vm.name}'}}")
Expand All @@ -362,9 +358,7 @@ def compare_network_traffic_bytes_and_metrics(
)
for entry in metric_result:
entry_value = entry.get("value")[1]
if math.isclose(
int(entry_value), int(packet_received[f"{entry.get('metric').get('type')}_bytes"]), rel_tol=0.05
):
if int(entry_value) >= int(network_packet_received[f"{entry.get('metric').get('type')}_bytes"]):
rx_tx_indicator = True
else:
break
Expand All @@ -374,15 +368,15 @@ def compare_network_traffic_bytes_and_metrics(


def validate_network_traffic_metrics_value(
prometheus: Prometheus, vm: VirtualMachineForTests, interface_name: str
prometheus: Prometheus, vm: VirtualMachineForTests, network_packet_received: dict[str, str]
) -> None:
samples = TimeoutSampler(
wait_timeout=TIMEOUT_4MIN,
sleep=TIMEOUT_10SEC,
func=compare_network_traffic_bytes_and_metrics,
prometheus=prometheus,
vm=vm,
vm_interface_name=interface_name,
network_packet_received=network_packet_received,
)
try:
for sample in samples:
Expand Down Expand Up @@ -692,14 +686,14 @@ def create_windows11_wsl2_vm(
artifactory_secret = get_artifactory_secret(namespace=namespace)
artifactory_config_map = get_artifactory_config_map(namespace=namespace)
dv = DataVolume(
client=client,
name=dv_name,
namespace=namespace,
storage_class=storage_class,
source="http",
url=get_http_image_url(image_directory=Images.Windows.DIR, image_name=Images.Windows.WIN11_WSL2_IMG),
size=Images.Windows.DEFAULT_DV_SIZE,
client=client,
api_name="storage",
source=REGISTRY_STR,
size=Images.Windows.CONTAINER_DISK_DV_SIZE,
storage_class=storage_class,
url=f"{get_test_artifact_server_url(schema=REGISTRY_STR)}/docker/windows-qe/win_11:virtio",
secret=artifactory_secret,
Comment thread
OhadRevah marked this conversation as resolved.
cert_configmap=artifactory_config_map.name,
)
Expand All @@ -709,15 +703,17 @@ def create_windows11_wsl2_vm(
name=vm_name,
namespace=namespace,
client=client,
vm_instance_type=VirtualMachineClusterInstancetype(client=client, name="u1.xlarge"),
vm_instance_type=VirtualMachineClusterInstancetype(client=client, name="u1.large"),
vm_preference=VirtualMachineClusterPreference(client=client, name="windows.11"),
data_volume_template={"metadata": dv.res["metadata"], "spec": dv.res["spec"]},
) as vm:
running_vm(vm=vm)
yield vm
cleanup_artifactory_secret_and_config_map(
artifactory_secret=artifactory_secret, artifactory_config_map=artifactory_config_map
)
try:
running_vm(vm=vm, dv_wait_timeout=TIMEOUT_40MIN)
yield vm
finally:
cleanup_artifactory_secret_and_config_map(
artifactory_secret=artifactory_secret, artifactory_config_map=artifactory_config_map
)


def get_vm_comparison_info_dict(vm: VirtualMachineForTests) -> dict[str, str]:
Expand Down Expand Up @@ -763,7 +759,10 @@ def get_pvc_size_bytes(vm: VirtualMachineForTests) -> str:


def validate_metric_value_greater_than_initial_value(
prometheus: Prometheus, metric_name: str, initial_value: float, timeout: int = TIMEOUT_4MIN
prometheus: Prometheus,
metric_name: str,
initial_value: float,
timeout: int = TIMEOUT_4MIN,
) -> None:
samples = TimeoutSampler(
wait_timeout=timeout,
Expand Down