Skip to content

Commit 9d8c607

Browse files
Millefeuille42Millefeuille42
authored
Merge pull request #324 from xcp-ng/test-disk-performance
feat(tests): add disk performance benchmark tests
2 parents 4580df1 + 13748c8 commit 9d8c607

5 files changed

Lines changed: 467 additions & 0 deletions

File tree

‎jobs.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,19 @@ class JobData(TypedDict):
255255
"markers": "quicktest and not unused_4k_disks",
256256
"name_filter": "not linstor and not zfsvol",
257257
},
258+
"storage-benchmarks": {
259+
"description": "runs disk benchmark tests",
260+
"requirements": [
261+
"A local SR on host A1"
262+
"A small VM that can be imported on the SR",
263+
"Enough storage space to store the largest test file (numjobs*memory*2)G"
264+
],
265+
"nb_pools": 1,
266+
"params": {
267+
"--vm": "single/small_vm",
268+
},
269+
"paths": ["tests/storage/benchmarks"],
270+
},
258271
"linstor-main": {
259272
"description": "tests the linstor storage driver, but avoids migrations and reboots",
260273
"requirements": [

‎tests/storage/benchmarks/__init__.py‎

Whitespace-only changes.

‎tests/storage/benchmarks/conftest.py‎

Lines changed: 131 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
import pytest
2+
3+
import logging
4+
import os
5+
import tempfile
6+
import urllib.request
7+
from datetime import datetime
8+
from pathlib import Path
9+
from urllib.parse import urlparse
10+
from uuid import uuid4
11+
12+
from lib.common import GiB, PackageManagerEnum
13+
from lib.host import Host
14+
from lib.sr import SR
15+
from lib.vbd import VBD
16+
from lib.vdi import VDI, ImageFormat
17+
from lib.vm import VM
18+
19+
from .helpers import FioBenchmarkCSV, load_results_from_csv
20+
21+
from typing import Generator, assert_never
22+
23+
MAX_LENGTH = 64 * GiB
24+
25+
26+
@pytest.fixture(scope="module")
27+
def running_unix_vm_with_fio(running_unix_vm: VM) -> Generator[VM, None, None]:
28+
vm = running_unix_vm
29+
snapshot = vm.snapshot()
30+
31+
package_manager = vm.detect_package_manager()
32+
match package_manager:
33+
case PackageManagerEnum.APT_GET:
34+
vm.ssh("apt-get update && apt install -y fio")
35+
case PackageManagerEnum.YUM:
36+
vm.ssh("yum install -y fio")
37+
case PackageManagerEnum.DNF:
38+
vm.ssh("dnf install -y fio")
39+
case PackageManagerEnum.ZYPPER:
40+
vm.ssh("zypper install -y fio")
41+
case PackageManagerEnum.APK:
42+
vm.ssh("apk add fio")
43+
case PackageManagerEnum.UNKNOWN:
44+
raise RuntimeError("Unsupported package manager: could not install fio")
45+
case _:
46+
assert_never(package_manager)
47+
48+
yield vm
49+
50+
# teardown
51+
try:
52+
snapshot.revert()
53+
finally:
54+
snapshot.destroy()
55+
56+
57+
@pytest.fixture(scope="function")
58+
def vdi_on_local_sr(host: Host, local_sr_on_hostA1: SR, image_format: ImageFormat) -> Generator[VDI, None, None]:
59+
sr = local_sr_on_hostA1
60+
vdi = sr.create_vdi("testVDI", MAX_LENGTH, image_format=image_format)
61+
logging.info(f">> Created VDI {vdi.uuid} of type {image_format}")
62+
63+
yield vdi
64+
65+
# teardown
66+
logging.info(f"<< Destroying VDI {vdi.uuid}")
67+
vdi.destroy()
68+
69+
70+
@pytest.fixture(scope="function")
71+
def plugged_vbd(vdi_on_local_sr: VDI, running_unix_vm_with_fio: VM) -> Generator[VBD, None, None]:
72+
vm = running_unix_vm_with_fio
73+
vdi = vdi_on_local_sr
74+
vbd = vm.create_vbd("autodetect", vdi.uuid)
75+
76+
logging.info(f">> Plugging VDI {vdi.uuid} on VM {vm.uuid}")
77+
vbd.plug()
78+
79+
yield vbd
80+
81+
# teardown
82+
logging.info(f"<< Unplugging VDI {vdi.uuid} from VM {vm.uuid}")
83+
vbd.unplug()
84+
vbd.destroy()
85+
86+
87+
@pytest.fixture(scope="module")
88+
def local_temp_dir() -> Generator[Path, None, None]:
89+
with tempfile.TemporaryDirectory() as tmpdir:
90+
yield Path(tmpdir)
91+
92+
93+
@pytest.fixture(scope="module")
94+
def result_csv_file(local_temp_dir: Path) -> Path:
95+
return local_temp_dir / f"results_{datetime.now().strftime('%Y-%m-%d_%H-%M-%S')}.csv"
96+
97+
98+
@pytest.fixture(scope="module")
99+
def temp_dir(running_unix_vm_with_fio: VM) -> Generator[Path, None, None]:
100+
vm = running_unix_vm_with_fio
101+
tmpdir = vm.ssh("mktemp -d")
102+
103+
yield Path(tmpdir)
104+
105+
# teardown
106+
vm.ssh(f"rm -r {tmpdir}")
107+
108+
109+
def pytest_addoption(parser: pytest.Parser) -> None:
110+
parser.addoption(
111+
"--benchmark-previous-results",
112+
action="store",
113+
default=None,
114+
help="Path/URI to previous CSV results file for comparison",
115+
)
116+
117+
118+
@pytest.fixture(scope="session")
119+
def prev_results(pytestconfig: pytest.Config) -> dict[str, list[FioBenchmarkCSV]]:
120+
csv_uri = pytestconfig.getoption("--benchmark-previous-results")
121+
if not csv_uri:
122+
return {}
123+
csv_path = csv_uri
124+
if urlparse(csv_uri).scheme != "":
125+
logging.info("Detected CSV path as an url")
126+
csv_path = f"/tmp/{uuid4()}.csv"
127+
urllib.request.urlretrieve(csv_uri, csv_path)
128+
logging.info(f"Fetching CSV file from {csv_uri} to {csv_path}")
129+
if not os.path.exists(csv_path):
130+
raise FileNotFoundError(csv_path)
131+
return load_results_from_csv(csv_path)

‎tests/storage/benchmarks/helpers.py‎

Lines changed: 181 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,181 @@
1+
import csv
2+
import os
3+
import statistics
4+
from collections import defaultdict
5+
from datetime import datetime
6+
from pathlib import Path
7+
8+
from pydantic import BaseModel, ConfigDict, Field, field_validator
9+
10+
from typing import Any, Literal
11+
12+
FIOTestMode = Literal["read", "randread", "write", "randwrite"]
13+
14+
15+
class FioLatency(BaseModel):
16+
min: float
17+
max: float
18+
mean: float
19+
stddev: float
20+
N: float
21+
percentile: dict[float, int] | None = None
22+
23+
24+
class FioSync(BaseModel):
25+
total_ios: int
26+
lat_ns: FioLatency
27+
28+
29+
class FioStats(BaseModel):
30+
io_bytes: int
31+
io_kbytes: int
32+
bw_bytes: int
33+
bw: int
34+
iops: float
35+
runtime: int
36+
total_ios: int
37+
short_ios: int
38+
drop_ios: int
39+
slat_ns: FioLatency
40+
clat_ns: FioLatency
41+
lat_ns: FioLatency
42+
bw_min: int
43+
bw_max: int
44+
bw_agg: float
45+
bw_mean: float
46+
bw_dev: float
47+
bw_samples: int
48+
iops_min: int
49+
iops_max: int
50+
iops_mean: float
51+
iops_stddev: float
52+
iops_samples: int
53+
54+
55+
class FioJobOptions(BaseModel):
56+
model_config = ConfigDict(extra="allow")
57+
58+
name: str
59+
rw: FIOTestMode
60+
bs: str
61+
iodepth: str
62+
size: str
63+
filename: Path
64+
direct: bool | None = None
65+
end_fsync: bool | None = None
66+
fsync_on_close: bool | None = None
67+
numjobs: int | None = None
68+
69+
70+
class FioJob(BaseModel):
71+
jobname: str
72+
groupid: int
73+
error: int
74+
eta: int
75+
elapsed: int
76+
job_options: FioJobOptions = Field(alias="job options")
77+
read: FioStats
78+
write: FioStats
79+
trim: FioStats
80+
sync: FioSync
81+
job_runtime: int
82+
usr_cpu: float
83+
sys_cpu: float
84+
ctx: int
85+
majf: int
86+
minf: int
87+
iodepth_level: dict[str, float]
88+
iodepth_submit: dict[str, float]
89+
iodepth_complete: dict[str, float]
90+
latency_ns: dict[str, float]
91+
latency_us: dict[str, float]
92+
latency_ms: dict[str, float]
93+
latency_depth: int
94+
latency_target: int
95+
latency_percentile: float
96+
latency_window: int
97+
98+
99+
class FioDiskUtil(BaseModel):
100+
name: str
101+
read_ios: int
102+
write_ios: int
103+
read_merges: int
104+
write_merges: int
105+
read_ticks: int
106+
write_ticks: int
107+
in_queue: int
108+
util: float
109+
110+
111+
class FioResultJson(BaseModel):
112+
fio_version: str = Field(alias="fio version")
113+
timestamp: datetime
114+
timestamp_ms: datetime
115+
time: datetime
116+
jobs: list[FioJob]
117+
disk_util: list[FioDiskUtil]
118+
119+
@field_validator("time", mode="before")
120+
@staticmethod
121+
def parse_fio_time_string(value: str) -> datetime:
122+
return datetime.strptime(value, "%a %b %d %H:%M:%S %Y")
123+
124+
125+
class FioBenchmarkCSV(BaseModel):
126+
timestamp: datetime
127+
test_name: str = Field(alias="test")
128+
mode: FIOTestMode
129+
bandwidth_mbps: float = Field(alias="bw_MBps")
130+
iops: float = Field(alias="IOPS")
131+
latency: float
132+
133+
134+
def log_result_csv(
135+
test_type: str,
136+
rw_mode: FIOTestMode,
137+
result_json: FioResultJson,
138+
csv_path: Path | str
139+
) -> FioBenchmarkCSV:
140+
assert len(result_json.jobs) >= 1
141+
142+
op_data: FioStats = getattr(result_json.jobs[0], rw_mode.replace("rand", ""))
143+
benchmark = FioBenchmarkCSV(
144+
timestamp=datetime.now(),
145+
test=test_type,
146+
mode=rw_mode,
147+
bw_MBps=round(op_data.bw / 1024, 2),
148+
IOPS=round(op_data.iops, 2),
149+
latency=round(op_data.lat_ns.mean, 2),
150+
)
151+
152+
result = benchmark.model_dump()
153+
file_exists = os.path.exists(csv_path)
154+
with open(csv_path, "a", newline="") as f:
155+
writer = csv.DictWriter(f, fieldnames=result.keys())
156+
if not file_exists:
157+
writer.writeheader()
158+
writer.writerow(result)
159+
160+
return benchmark
161+
162+
163+
def load_results_from_csv(csv_path: Path | str) -> dict[str, list[FioBenchmarkCSV]]:
164+
results: dict[str, list[FioBenchmarkCSV]] = defaultdict(list)
165+
with open(csv_path, newline="") as f:
166+
reader = csv.DictReader(f)
167+
for row in reader:
168+
results[row["test"]].append(FioBenchmarkCSV.model_validate(row))
169+
return dict(results)
170+
171+
172+
def mean(data: list[FioBenchmarkCSV], key: str) -> float:
173+
values = [
174+
float(val) for x in data
175+
if (val := getattr(x, key, None)) is not None
176+
]
177+
178+
if not values:
179+
return 0.0
180+
181+
return statistics.mean(values)

0 commit comments

Comments
 (0)