Skip to content

Commit 92d557e

Browse files
zh-jqcursoragent
zh-jq
and
cursoragent
committed
vey-bench: assert --json-file output in coverage targets
Add shared jq helpers and per-target JSON shape checks so coverage exercises machine-readable results across h1/h2/h3 and other targets. Co-authored-by: Cursor <cursoragent@cursor.com>
1 parent e907a59 commit 92d557e

12 files changed

Lines changed: 228 additions & 0 deletions

‎scripts/coverage/vey-bench/.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,3 @@
11

22
*.pem
3+
.json-out/

‎scripts/coverage/vey-bench/json_assert.sh‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
2+
# Shared helpers for --json-file result assertions (POSIX sh)
3+
4+
JSON_OUT_DIR="${RUN_DIR}/.json-out"
5+
mkdir -p "${JSON_OUT_DIR}"
6+
7+
assert_json_eq()
8+
{
9+
_aj_eq_file=$1
10+
_aj_eq_expr=$2
11+
_aj_eq_expected=$3
12+
_aj_eq_actual=$(jq -er "${_aj_eq_expr}" "${_aj_eq_file}")
13+
if [ "${_aj_eq_actual}" != "${_aj_eq_expected}" ]; then
14+
echo "JSON assert failed for ${_aj_eq_file}: ${_aj_eq_expr} expected='${_aj_eq_expected}' actual='${_aj_eq_actual}'" >&2
15+
exit 1
16+
fi
17+
}
18+
19+
assert_json_gt()
20+
{
21+
_aj_gt_file=$1
22+
_aj_gt_expr=$2
23+
_aj_gt_min=$3
24+
_aj_gt_actual=$(jq -er "${_aj_gt_expr}" "${_aj_gt_file}")
25+
if ! [ "${_aj_gt_actual}" -gt "${_aj_gt_min}" ]; then
26+
echo "JSON assert failed for ${_aj_gt_file}: ${_aj_gt_expr} expected > ${_aj_gt_min} actual='${_aj_gt_actual}'" >&2
27+
exit 1
28+
fi
29+
}
30+
31+
assert_json_type()
32+
{
33+
_aj_ty_file=$1
34+
_aj_ty_expr=$2
35+
_aj_ty_type=$3
36+
_aj_ty_actual=$(jq -er "${_aj_ty_expr} | type" "${_aj_ty_file}")
37+
if [ "${_aj_ty_actual}" != "${_aj_ty_type}" ]; then
38+
echo "JSON assert failed for ${_aj_ty_file}: ${_aj_ty_expr} type expected='${_aj_ty_type}' actual='${_aj_ty_actual}'" >&2
39+
exit 1
40+
fi
41+
}
42+
43+
assert_json_null()
44+
{
45+
_aj_nu_file=$1
46+
_aj_nu_expr=$2
47+
_aj_nu_actual=$(jq -er "${_aj_nu_expr} | type" "${_aj_nu_file}")
48+
if [ "${_aj_nu_actual}" != "null" ]; then
49+
echo "JSON assert failed for ${_aj_nu_file}: ${_aj_nu_expr} expected null actual type='${_aj_nu_actual}'" >&2
50+
exit 1
51+
fi
52+
}
53+
54+
assert_hist_snapshot()
55+
{
56+
_aj_hs_file=$1
57+
_aj_hs_expr=$2
58+
assert_json_type "${_aj_hs_file}" "${_aj_hs_expr}.min" number
59+
assert_json_type "${_aj_hs_file}" "${_aj_hs_expr}.mean" number
60+
assert_json_type "${_aj_hs_file}" "${_aj_hs_expr}.stdev" number
61+
assert_json_type "${_aj_hs_file}" "${_aj_hs_expr}.p90" number
62+
assert_json_type "${_aj_hs_file}" "${_aj_hs_expr}.max" number
63+
}
64+
65+
assert_json_report()
66+
{
67+
# assert_json_report <file> <target> <concurrency> <complete_requests>
68+
_aj_rp_file=$1
69+
_aj_rp_target=$2
70+
_aj_rp_concurrency=$3
71+
_aj_rp_complete=$4
72+
assert_json_eq "${_aj_rp_file}" .version 1
73+
assert_json_eq "${_aj_rp_file}" .target "${_aj_rp_target}"
74+
assert_json_eq "${_aj_rp_file}" .concurrency "${_aj_rp_concurrency}"
75+
assert_json_eq "${_aj_rp_file}" .global.complete_requests "${_aj_rp_complete}"
76+
assert_json_eq "${_aj_rp_file}" .global.failed_requests 0
77+
assert_json_gt "${_aj_rp_file}" .global.total_time_ns 0
78+
assert_json_type "${_aj_rp_file}" .global.requests_per_sec number
79+
assert_json_type "${_aj_rp_file}" .histograms object
80+
assert_hist_snapshot "${_aj_rp_file}" .histograms.durations_ns.total
81+
assert_json_type "${_aj_rp_file}" .percentiles_ns.p50 number
82+
assert_json_type "${_aj_rp_file}" .percentiles_ns.p100 number
83+
}
84+
85+
assert_json_tcp_traffic()
86+
{
87+
_aj_tcp_file=$1
88+
assert_json_type "${_aj_tcp_file}" .traffic.tcp object
89+
assert_json_gt "${_aj_tcp_file}" .traffic.tcp.send_bytes 0
90+
assert_json_gt "${_aj_tcp_file}" .traffic.tcp.recv_bytes 0
91+
assert_json_null "${_aj_tcp_file}" .traffic.udp
92+
}
93+
94+
assert_json_udp_traffic()
95+
{
96+
_aj_udp_file=$1
97+
assert_json_type "${_aj_udp_file}" .traffic.udp object
98+
assert_json_gt "${_aj_udp_file}" .traffic.udp.send_bytes 0
99+
assert_json_gt "${_aj_udp_file}" .traffic.udp.send_packets 0
100+
assert_json_gt "${_aj_udp_file}" .traffic.udp.recv_packets 0
101+
assert_json_null "${_aj_udp_file}" .traffic.tcp
102+
}
103+
104+
assert_json_http_histograms()
105+
{
106+
_aj_hh_file=$1
107+
assert_hist_snapshot "${_aj_hh_file}" .histograms.conn_used_times
108+
assert_hist_snapshot "${_aj_hh_file}" .histograms.durations_ns.connect
109+
assert_hist_snapshot "${_aj_hh_file}" .histograms.durations_ns.send_hdr
110+
assert_hist_snapshot "${_aj_hh_file}" .histograms.durations_ns.send_all
111+
assert_hist_snapshot "${_aj_hh_file}" .histograms.durations_ns.recv_hdr
112+
assert_hist_snapshot "${_aj_hh_file}" .histograms.durations_ns.total
113+
}

‎scripts/coverage/vey-bench/run.sh‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,8 @@ vey_bench()
2727
"${PROJECT_DIR}"/target/debug/vey-bench --no-progress-bar --log-error 1 "$@"
2828
}
2929

30+
. "${RUN_DIR}"/json_assert.sh
31+
3032
set -x
3133

3234
"${PROJECT_DIR}"/target/debug/vey-bench version
@@ -49,4 +51,5 @@ kill -INT $STATSD_PID
4951

5052
# cleanup
5153

54+
rm -rf "${JSON_OUT_DIR}"
5255
docker compose -f "${PROJECT_DIR}"/scripts/coverage/vey-bench/docker-compose.yml down

‎scripts/coverage/vey-bench/target_dns.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,3 +16,13 @@ vey_bench dns "94.140.14.140" -e doq www.example.com,A --dump-result
1616

1717
# Dns over Http/3, via AdGuard Public DNS
1818
vey_bench dns "94.140.14.140" -e doh3 www.example.com,A --dump-result
19+
20+
# JSON result
21+
22+
JSON_FILE="${JSON_OUT_DIR}/dns-tcp.json"
23+
vey_bench dns "1.1.1.1" --tcp www.example.com,A -n 2 -c 1 --json-file "${JSON_FILE}"
24+
assert_json_report "${JSON_FILE}" dns 1 2
25+
assert_json_type "${JSON_FILE}" .connections object
26+
assert_json_null "${JSON_FILE}" .tls
27+
assert_json_null "${JSON_FILE}" .histograms.conn_used_times
28+
assert_json_null "${JSON_FILE}" .histograms.durations_ns.connect

‎scripts/coverage/vey-bench/target_h1.sh‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,3 +64,32 @@ test_https_get https://httpbin.local:2443/get
6464

6565
test_https_post https://httpbin.local:9443/post
6666
test_https_post https://httpbin.local:2443/post
67+
68+
# JSON result
69+
70+
JSON_FILE="${JSON_OUT_DIR}/h1-http.json"
71+
vey_bench h1 http://httpbin.local/get --ok-status 200 -n 5 -c 2 --json-file "${JSON_FILE}"
72+
assert_json_report "${JSON_FILE}" h1 2 5
73+
assert_json_type "${JSON_FILE}" .global.requests_distribution object
74+
assert_json_type "${JSON_FILE}" .connections object
75+
assert_json_tcp_traffic "${JSON_FILE}"
76+
assert_json_null "${JSON_FILE}" .tls
77+
assert_json_http_histograms "${JSON_FILE}"
78+
79+
JSON_FILE="${JSON_OUT_DIR}/h1-https.json"
80+
vey_bench h1 https://httpbin.local:9443/get --ok-status 200 --tls-ca-cert "${TEST_CA_CERT_FILE}" -n 3 -c 1 --json-file "${JSON_FILE}"
81+
assert_json_report "${JSON_FILE}" h1 1 3
82+
assert_json_null "${JSON_FILE}" .global.requests_distribution
83+
assert_json_tcp_traffic "${JSON_FILE}"
84+
assert_json_type "${JSON_FILE}" .tls.target object
85+
assert_json_gt "${JSON_FILE}" .tls.target.total 0
86+
assert_json_null "${JSON_FILE}" .tls.proxy
87+
assert_json_http_histograms "${JSON_FILE}"
88+
89+
JSON_FILE="${JSON_OUT_DIR}/h1-no-summary.json"
90+
OUT=$(vey_bench h1 http://httpbin.local/get --ok-status 200 -n 2 --no-summary --json-file "${JSON_FILE}")
91+
[ -z "${OUT}" ] || {
92+
echo "expected empty stdout with --no-summary, got: ${OUT}" >&2
93+
exit 1
94+
}
95+
assert_json_report "${JSON_FILE}" h1 1 2

‎scripts/coverage/vey-bench/target_h2.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,13 @@ URL=https://httpbin.local:2443/post
2020
vey_bench h2 "${URL}" --method POST --payload 31323334 --ok-status 200 --tls-ca-cert "${TEST_CA_CERT_FILE}"
2121
vey_bench h2 "${URL}" --method POST --payload 31323334 --binary --ok-status 200 --tls-ca-cert "${TEST_CA_CERT_FILE}"
2222
vey_bench h2 "${URL}" --method POST --payload name=foo -H "Content-Type: application/x-www-form-urlencoded" --ok-status 200 --tls-ca-cert "${TEST_CA_CERT_FILE}"
23+
24+
# JSON result
25+
26+
JSON_FILE="${JSON_OUT_DIR}/h2.json"
27+
vey_bench h2 https://httpbin.local:2443/get --ok-status 200 --tls-ca-cert "${TEST_CA_CERT_FILE}" -n 3 -c 1 --json-file "${JSON_FILE}"
28+
assert_json_report "${JSON_FILE}" h2 1 3
29+
assert_json_type "${JSON_FILE}" .connections object
30+
assert_json_tcp_traffic "${JSON_FILE}"
31+
assert_json_type "${JSON_FILE}" .tls.target object
32+
assert_json_http_histograms "${JSON_FILE}"

‎scripts/coverage/vey-bench/target_h3.sh‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,3 +21,13 @@ vey_bench h3 "${URL}" --method POST --payload 31323334 --binary --ok-status 200
2121
vey_bench h3 "${URL}" --method POST --payload name=foo -H "Content-Type: application/x-www-form-urlencoded" --ok-status 200 --tls-ca-cert "${TEST_CA_CERT_FILE}"
2222

2323
done
24+
25+
# JSON result
26+
27+
JSON_FILE="${JSON_OUT_DIR}/h3.json"
28+
vey_bench h3 https://httpbin.local:2443/get --ok-status 200 --tls-ca-cert "${TEST_CA_CERT_FILE}" -n 3 -c 1 --json-file "${JSON_FILE}"
29+
assert_json_report "${JSON_FILE}" h3 1 3
30+
assert_json_type "${JSON_FILE}" .connections object
31+
assert_json_udp_traffic "${JSON_FILE}"
32+
assert_json_null "${JSON_FILE}" .tls
33+
assert_json_http_histograms "${JSON_FILE}"

‎scripts/coverage/vey-bench/target_keyless_openssl.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,3 +25,14 @@ vey_bench -c 2 -l 5 -r 10/100ms -t 4 ${TARGET_PARAMS}
2525
vey_bench -c 2 -l 5 -r 100/s -t 4 ${TARGET_PARAMS}
2626
vey_bench -c 2 -l 5 -r 100 -t 4 ${TARGET_PARAMS}
2727
vey_bench -c 1 -t 4 --unaided --emit-metrics ${TARGET_PARAMS}
28+
29+
# JSON result
30+
JSON_FILE="${JSON_OUT_DIR}/keyless-openssl.json"
31+
vey_bench -c 2 -n 4 --json-file "${JSON_FILE}" ${TARGET_PARAMS}
32+
assert_json_report "${JSON_FILE}" keyless/openssl 2 4
33+
assert_json_type "${JSON_FILE}" .global.requests_distribution object
34+
assert_json_null "${JSON_FILE}" .connections
35+
assert_json_null "${JSON_FILE}" .traffic
36+
assert_json_null "${JSON_FILE}" .tls
37+
assert_json_null "${JSON_FILE}" .histograms.conn_used_times
38+
assert_json_null "${JSON_FILE}" .histograms.durations_ns.connect

‎scripts/coverage/vey-bench/target_openssl.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,14 @@
22
vey_bench openssl httpbin.local:9443 --tls-ca-cert "${TEST_CA_CERT_FILE}"
33

44
vey_bench openssl 127.0.0.1:9443 --tls-name httpbin.local --tls-ca-cert "${TEST_CA_CERT_FILE}"
5+
6+
# JSON result
7+
8+
JSON_FILE="${JSON_OUT_DIR}/openssl.json"
9+
vey_bench openssl httpbin.local:9443 --tls-ca-cert "${TEST_CA_CERT_FILE}" -n 3 -c 1 --json-file "${JSON_FILE}"
10+
assert_json_report "${JSON_FILE}" openssl 1 3
11+
assert_json_type "${JSON_FILE}" .connections object
12+
assert_json_tcp_traffic "${JSON_FILE}"
13+
assert_json_type "${JSON_FILE}" .tls.target object
14+
assert_json_null "${JSON_FILE}" .histograms.conn_used_times
15+
assert_json_null "${JSON_FILE}" .histograms.durations_ns.connect

‎scripts/coverage/vey-bench/target_rustls.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,14 @@
22
vey_bench rustls httpbin.local:9443 --tls-ca-cert "${TEST_CA_CERT_FILE}"
33

44
vey_bench rustls 127.0.0.1:9443 --tls-name httpbin.local --tls-ca-cert "${TEST_CA_CERT_FILE}"
5+
6+
# JSON result
7+
8+
JSON_FILE="${JSON_OUT_DIR}/rustls.json"
9+
vey_bench rustls httpbin.local:9443 --tls-ca-cert "${TEST_CA_CERT_FILE}" -n 3 -c 1 --json-file "${JSON_FILE}"
10+
assert_json_report "${JSON_FILE}" rustls 1 3
11+
assert_json_type "${JSON_FILE}" .connections object
12+
assert_json_tcp_traffic "${JSON_FILE}"
13+
assert_json_type "${JSON_FILE}" .tls.target object
14+
assert_json_null "${JSON_FILE}" .histograms.conn_used_times
15+
assert_json_null "${JSON_FILE}" .histograms.durations_ns.connect

0 commit comments

Comments
 (0)