feat(tests): add disk performance benchmark tests

Signed-off-by: Mathieu Labourier <mathieu.labourier@vates.tech>

Author: Millefeuille42

jobs.py

@@ -255,6 +255,19 @@ class JobData(TypedDict):
         "markers": "quicktest and not unused_4k_disks",
         "name_filter": "not linstor and not zfsvol",
     },
+    "storage-benchmarks": {
+        "description": "runs disk benchmark tests",
+        "requirements": [
+            "A local SR on host A1"
+            "A small VM that can be imported on the SR",
+            "Enough storage space to store the largest test file (numjobs*memory*2)G"
+        ],
+        "nb_pools": 1,
+        "params": {
+            "--vm": "single/small_vm",
+        },
+        "paths": ["tests/storage/benchmarks"],
+    },
     "linstor-main": {
         "description": "tests the linstor storage driver, but avoids migrations and reboots",
         "requirements": [

tests/storage/benchmarks/__init__.py

@@ -0,0 +1,131 @@
+import pytest
+
+import logging
+import os
+import tempfile
+import urllib.request
+from datetime import datetime
+from pathlib import Path
+from urllib.parse import urlparse
+from uuid import uuid4
+
+from lib.common import GiB, PackageManagerEnum
+from lib.host import Host
+from lib.sr import SR
+from lib.vbd import VBD
+from lib.vdi import VDI, ImageFormat
+from lib.vm import VM
+
+from .helpers import FioBenchmarkCSV, load_results_from_csv
+
+from typing import Generator, assert_never
+
+MAX_LENGTH = 64 * GiB
+
+
+@pytest.fixture(scope="module")
+def running_unix_vm_with_fio(running_unix_vm: VM) -> Generator[VM, None, None]:
+    vm = running_unix_vm
+    snapshot = vm.snapshot()
+
+    package_manager = vm.detect_package_manager()
+    match package_manager:
+        case PackageManagerEnum.APT_GET:
+            vm.ssh("apt-get update && apt install -y fio")
+        case PackageManagerEnum.YUM:
+            vm.ssh("yum install -y fio")
+        case PackageManagerEnum.DNF:
+            vm.ssh("dnf install -y fio")
+        case PackageManagerEnum.ZYPPER:
+            vm.ssh("zypper install -y fio")
+        case PackageManagerEnum.APK:
+            vm.ssh("apk add fio")
+        case PackageManagerEnum.UNKNOWN:
+            raise RuntimeError("Unsupported package manager: could not install fio")
+        case _:
+            assert_never(package_manager)
+
+    yield vm
+
+    # teardown
+    try:
+        snapshot.revert()
+    finally:
+        snapshot.destroy()
+
+
+@pytest.fixture(scope="function")
+def vdi_on_local_sr(host: Host, local_sr_on_hostA1: SR, image_format: ImageFormat) -> Generator[VDI, None, None]:
+    sr = local_sr_on_hostA1
+    vdi = sr.create_vdi("testVDI", MAX_LENGTH, image_format=image_format)
+    logging.info(f">> Created VDI {vdi.uuid} of type {image_format}")
+
+    yield vdi
+
+    # teardown
+    logging.info(f"<< Destroying VDI {vdi.uuid}")
+    vdi.destroy()
+
+
+@pytest.fixture(scope="function")
+def plugged_vbd(vdi_on_local_sr: VDI, running_unix_vm_with_fio: VM) -> Generator[VBD, None, None]:
+    vm = running_unix_vm_with_fio
+    vdi = vdi_on_local_sr
+    vbd = vm.create_vbd("autodetect", vdi.uuid)
+
+    logging.info(f">> Plugging VDI {vdi.uuid} on VM {vm.uuid}")
+    vbd.plug()
+
+    yield vbd
+
+    # teardown
+    logging.info(f"<< Unplugging VDI {vdi.uuid} from VM {vm.uuid}")
+    vbd.unplug()
+    vbd.destroy()
+
+
+@pytest.fixture(scope="module")
+def local_temp_dir() -> Generator[Path, None, None]:
+    with tempfile.TemporaryDirectory() as tmpdir:
+        yield Path(tmpdir)
+
+
+@pytest.fixture(scope="module")
+def result_csv_file(local_temp_dir: Path) -> Path:
+    return local_temp_dir / f"results_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv"
+
+
+@pytest.fixture(scope="module")
+def temp_dir(running_unix_vm_with_fio: VM) -> Generator[Path, None, None]:
+    vm = running_unix_vm_with_fio
+    tmpdir = vm.ssh("mktemp -d")
+
+    yield Path(tmpdir)
+
+    # teardown
+    vm.ssh(f"rm -r {tmpdir}")
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--benchmark-previous-results",
+        action="store",
+        default=None,
+        help="Path/URI to previous CSV results file for comparison",
+    )
+
+
+@pytest.fixture(scope="session")
+def prev_results(pytestconfig: pytest.Config) -> dict[str, list[FioBenchmarkCSV]]:
+    csv_uri = pytestconfig.getoption("--benchmark-previous-results")
+    if not csv_uri:
+        return {}
+    csv_path = csv_uri
+    if urlparse(csv_uri).scheme != "":
+        logging.info("Detected CSV path as an url")
+        csv_path = f"/tmp/{uuid4()}.csv"
+        urllib.request.urlretrieve(csv_uri, csv_path)
+        logging.info(f"Fetching CSV file from {csv_uri} to {csv_path}")
+    if not os.path.exists(csv_path):
+        raise FileNotFoundError(csv_path)
+    return load_results_from_csv(csv_path)

tests/storage/benchmarks/conftest.py

@@ -0,0 +1,181 @@
+import csv
+import os
+import statistics
+from collections import defaultdict
+from datetime import datetime
+from pathlib import Path
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator
+
+from typing import Any, Literal
+
+FIOTestMode = Literal["read", "randread", "write", "randwrite"]
+
+
+class FioLatency(BaseModel):
+    min: float
+    max: float
+    mean: float
+    stddev: float
+    N: float
+    percentile: dict[float, int] | None = None
+
+
+class FioSync(BaseModel):
+    total_ios: int
+    lat_ns: FioLatency
+
+
+class FioStats(BaseModel):
+    io_bytes: int
+    io_kbytes: int
+    bw_bytes: int
+    bw: int
+    iops: float
+    runtime: int
+    total_ios: int
+    short_ios: int
+    drop_ios: int
+    slat_ns: FioLatency
+    clat_ns: FioLatency
+    lat_ns: FioLatency
+    bw_min: int
+    bw_max: int
+    bw_agg: float
+    bw_mean: float
+    bw_dev: float
+    bw_samples: int
+    iops_min: int
+    iops_max: int
+    iops_mean: float
+    iops_stddev: float
+    iops_samples: int
+
+
+class FioJobOptions(BaseModel):
+    model_config = ConfigDict(extra="allow")
+
+    name: str
+    rw: FIOTestMode
+    bs: str
+    iodepth: str
+    size: str
+    filename: Path
+    direct: bool | None = None
+    end_fsync: bool | None = None
+    fsync_on_close: bool | None = None
+    numjobs: int | None = None
+
+
+class FioJob(BaseModel):
+    jobname: str
+    groupid: int
+    error: int
+    eta: int
+    elapsed: int
+    job_options: FioJobOptions = Field(alias="job options")
+    read: FioStats
+    write: FioStats
+    trim: FioStats
+    sync: FioSync
+    job_runtime: int
+    usr_cpu: float
+    sys_cpu: float
+    ctx: int
+    majf: int
+    minf: int
+    iodepth_level: dict[str, float]
+    iodepth_submit: dict[str, float]
+    iodepth_complete: dict[str, float]
+    latency_ns: dict[str, float]
+    latency_us: dict[str, float]
+    latency_ms: dict[str, float]
+    latency_depth: int
+    latency_target: int
+    latency_percentile: float
+    latency_window: int
+
+
+class FioDiskUtil(BaseModel):
+    name: str
+    read_ios: int
+    write_ios: int
+    read_merges: int
+    write_merges: int
+    read_ticks: int
+    write_ticks: int
+    in_queue: int
+    util: float
+
+
+class FioResultJson(BaseModel):
+    fio_version: str = Field(alias="fio version")
+    timestamp: datetime
+    timestamp_ms: datetime
+    time: datetime
+    jobs: list[FioJob]
+    disk_util: list[FioDiskUtil]
+
+    @field_validator("time", mode="before")
+    @staticmethod
+    def parse_fio_time_string(value: str) -> datetime:
+        return datetime.strptime(value, "%a %b %d %H:%M:%S %Y")
+
+
+class FioBenchmarkCSV(BaseModel):
+    timestamp: datetime
+    test_name: str = Field(alias="test")
+    mode: FIOTestMode
+    bandwidth_mbps: float = Field(alias="bw_MBps")
+    iops: float = Field(alias="IOPS")
+    latency: float
+
+
+def log_result_csv(
+        test_type: str,
+        rw_mode: FIOTestMode,
+        result_json: FioResultJson,
+        csv_path: Path | str
+) -> FioBenchmarkCSV:
+    assert len(result_json.jobs) >= 1
+
+    op_data: FioStats = getattr(result_json.jobs[0], rw_mode.replace("rand", ""))
+    benchmark = FioBenchmarkCSV(
+        timestamp=datetime.now(),
+        test=test_type,
+        mode=rw_mode,
+        bw_MBps=round(op_data.bw / 1024, 2),
+        IOPS=round(op_data.iops, 2),
+        latency=round(op_data.lat_ns.mean, 2),
+    )
+
+    result = benchmark.model_dump()
+    file_exists = os.path.exists(csv_path)
+    with open(csv_path, "a", newline="") as f:
+        writer = csv.DictWriter(f, fieldnames=result.keys())
+        if not file_exists:
+            writer.writeheader()
+        writer.writerow(result)
+
+    return benchmark
+
+
+def load_results_from_csv(csv_path: Path | str) -> dict[str, list[FioBenchmarkCSV]]:
+    results: dict[str, list[FioBenchmarkCSV]] = defaultdict(list)
+    with open(csv_path, newline="") as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            results[row["test"]].append(FioBenchmarkCSV.model_validate(row))
+    return dict(results)
+
+
+def mean(data: list[FioBenchmarkCSV], key: str) -> float:
+    values = [
+        float(val) for x in data
+        if (val := getattr(x, key, None)) is not None
+    ]
+
+    if not values:
+        return 0.0
+
+    return statistics.mean(values)