Skip to content

Commit 0dfea04

Browse files
cijothomascijothomas
committed
save work
1 parent 5557d98 commit 0dfea04

9 files changed

Lines changed: 297 additions & 165 deletions

File tree

‎.github/workflows/pipeline-perf-test-continuous.yml‎

Lines changed: 2 additions & 152 deletions
Original file line numberDiff line numberDiff line change
@@ -181,161 +181,11 @@ jobs:
181181
182182
- name: Compute scaling efficiency metrics
183183
run: |
184-
python3 << 'EOF'
185-
import json
186-
187-
with open('output-saturation.json') as f:
188-
data = json.load(f)
189-
190-
# Group by cores and protocol - collect throughput and CPU
191-
throughput = {}
192-
cpu_norm = {}
193-
for entry in data:
194-
extra = entry['extra']
195-
parts = extra.split(' - ')
196-
if len(parts) < 3:
197-
continue
198-
199-
cores = parts[2].split('/')[0].split()[0]
200-
protocol = extra.split('/')[1].split(' - ')[0] if '/' in extra else 'unknown'
201-
key = f"{cores}core-{protocol}"
202-
203-
if entry['name'] == 'logs_received_rate':
204-
throughput[key] = entry['value']
205-
elif entry['name'] == 'cpu_percentage_normalized_avg':
206-
cpu_norm[key] = entry['value']
207-
208-
# Calculate scaling metrics
209-
scaling_metrics = []
210-
for protocol in ['OTLP-ATTR-OTLP', 'OTAP-ATTR-OTLP']:
211-
baseline = throughput.get(f"1core-{protocol}", 1)
212-
213-
for cores in ['1', '2', '4', '8']:
214-
key = f"{cores}core-{protocol}"
215-
if key in throughput:
216-
cores_int = int(cores)
217-
actual_speedup = throughput[key] / baseline if baseline > 0 else 0
218-
ideal_speedup = cores_int
219-
efficiency = (actual_speedup / ideal_speedup) * 100 if ideal_speedup > 0 else 0
220-
per_core = throughput[key] / cores_int
221-
222-
# Add efficiency metric (higher is better)
223-
scaling_metrics.append({
224-
"name": "scaling_efficiency",
225-
"unit": "%",
226-
"value": efficiency,
227-
"extra": f"Continuous - Saturation - {cores} Core(s)/{protocol} - Scaling Efficiency"
228-
})
229-
230-
# Add per-core throughput (should remain constant for linear scaling)
231-
scaling_metrics.append({
232-
"name": "per_core_throughput",
233-
"unit": "logs/sec/core",
234-
"value": per_core,
235-
"extra": f"Continuous - Saturation - {cores} Core(s)/{protocol} - Per-Core Throughput"
236-
})
237-
238-
# Add speedup metric
239-
scaling_metrics.append({
240-
"name": "speedup",
241-
"unit": "x",
242-
"value": actual_speedup,
243-
"extra": f"Continuous - Saturation - {cores} Core(s)/{protocol} - Speedup vs 1-core"
244-
})
245-
246-
# Merge with original data
247-
data.extend(scaling_metrics)
248-
249-
with open('output-saturation.json', 'w') as f:
250-
json.dump(data, f, indent=2)
251-
252-
print(f"Added {len(scaling_metrics)} scaling metrics")
253-
EOF
184+
python3 .github/workflows/scripts/compute-scaling-metrics.py output-saturation.json output-saturation.json
254185
255186
- name: Generate scaling analysis summary
256187
run: |
257-
python3 << 'EOF'
258-
import json
259-
import os
260-
261-
with open('output-saturation.json') as f:
262-
data = json.load(f)
263-
264-
# Group metrics by configuration
265-
metrics = {}
266-
for entry in data:
267-
extra = entry['extra']
268-
parts = extra.split(' - ')
269-
if len(parts) < 3:
270-
continue
271-
272-
cores = parts[2].split('/')[0].split()[0]
273-
protocol = extra.split('/')[1].split(' - ')[0] if '/' in extra else 'unknown'
274-
key = f"{cores}core-{protocol}"
275-
276-
if key not in metrics:
277-
metrics[key] = {}
278-
279-
name = entry['name']
280-
if name == 'logs_received_rate':
281-
metrics[key]['throughput'] = entry['value']
282-
elif name == 'cpu_percentage_normalized_avg':
283-
metrics[key]['cpu'] = entry['value']
284-
elif name == 'speedup':
285-
metrics[key]['speedup'] = entry['value']
286-
elif name == 'scaling_efficiency':
287-
metrics[key]['efficiency'] = entry['value']
288-
elif name == 'dropped_logs_percentage':
289-
metrics[key]['dropped'] = entry['value']
290-
291-
# Write to GitHub Step Summary
292-
with open(os.environ['GITHUB_STEP_SUMMARY'], 'a') as f:
293-
f.write("\n## 🚀 Core Scaling Analysis\n\n")
294-
f.write("### OTLP Protocol\n\n")
295-
f.write("| Cores | Throughput (logs/s) | Speedup | Efficiency | CPU % | Dropped % |\n")
296-
f.write("|-------|--------------------:|--------:|-----------:|------:|----------:|\n")
297-
298-
for cores in ['1', '2', '4', '8']:
299-
key = f"{cores}core-OTLP-ATTR-OTLP"
300-
if key in metrics and 'throughput' in metrics[key]:
301-
m = metrics[key]
302-
speedup = m.get('speedup', 0)
303-
efficiency = m.get('efficiency', 0)
304-
cpu = m.get('cpu', 0)
305-
throughput = m.get('throughput', 0)
306-
dropped = m.get('dropped', 0)
307-
308-
# Add status emoji
309-
eff_emoji = "🟢" if efficiency >= 80 else "🟡" if efficiency >= 60 else "🔴"
310-
cpu_emoji = "✅" if cpu >= 90 else "⚠️" if cpu >= 70 else "❌"
311-
312-
f.write(f"| {cores} {eff_emoji} | {throughput:>15,.0f} | {speedup:>5.2f}x | {efficiency:>8.1f}% | {cpu:>4.1f}% {cpu_emoji} | {dropped:>6.2f}% |\n")
313-
314-
f.write("\n### OTAP Protocol\n\n")
315-
f.write("| Cores | Throughput (logs/s) | Speedup | Efficiency | CPU % | Dropped % |\n")
316-
f.write("|-------|--------------------:|--------:|-----------:|------:|----------:|\n")
317-
318-
for cores in ['1', '2', '4', '8']:
319-
key = f"{cores}core-OTAP-ATTR-OTLP"
320-
if key in metrics and 'throughput' in metrics[key]:
321-
m = metrics[key]
322-
speedup = m.get('speedup', 0)
323-
efficiency = m.get('efficiency', 0)
324-
cpu = m.get('cpu', 0)
325-
throughput = m.get('throughput', 0)
326-
dropped = m.get('dropped', 0)
327-
328-
# Add status emoji
329-
eff_emoji = "🟢" if efficiency >= 80 else "🟡" if efficiency >= 60 else "🔴"
330-
cpu_emoji = "✅" if cpu >= 90 else "⚠️" if cpu >= 70 else "❌"
331-
332-
f.write(f"| {cores} {eff_emoji} | {throughput:>15,.0f} | {speedup:>5.2f}x | {efficiency:>8.1f}% | {cpu:>4.1f}% {cpu_emoji} | {dropped:>6.2f}% |\n")
333-
334-
f.write("\n**Legend:**\n")
335-
f.write("- 🟢 Efficiency ≥80% | 🟡 60-80% | 🔴 <60%\n")
336-
f.write("- ✅ CPU ≥90% (saturated) | ⚠️ 70-90% | ❌ <70% (under-utilized)\n")
337-
f.write("- **Ideal Linear Scaling:** Efficiency = 100%, Speedup = # of cores\n\n")
338-
EOF
188+
python3 .github/workflows/scripts/generate-scaling-summary.py output-saturation.json
339189
340190
- name: Update pipeline benchmark data and deploy to GitHub Pages
341191
uses: benchmark-action/github-action-benchmark@d48d326b4ca9ba73ca0cd0d59f108f9e02a381c7

‎.github/workflows/scripts/compute-scaling-metrics.py‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Compute scaling efficiency metrics from benchmark data.
4+
5+
Reads benchmark JSON data and adds computed metrics:
6+
- scaling_efficiency: How close to ideal linear scaling (100% = perfect)
7+
- per_core_throughput: Throughput divided by number of cores
8+
- speedup: Actual speedup compared to 1-core baseline
9+
10+
Usage:
11+
python compute-scaling-metrics.py <input-file> <output-file>
12+
13+
Example:
14+
python compute-scaling-metrics.py output-saturation.json output-saturation-with-metrics.json
15+
"""
16+
17+
import json
18+
import sys
19+
20+
21+
def compute_scaling_metrics(input_file, output_file):
22+
"""Compute and add scaling metrics to benchmark data."""
23+
24+
with open(input_file) as f:
25+
data = json.load(f)
26+
27+
# Group by cores and protocol - collect throughput and CPU
28+
throughput = {}
29+
cpu_norm = {}
30+
for entry in data:
31+
extra = entry['extra']
32+
parts = extra.split(' - ')
33+
if len(parts) < 3:
34+
continue
35+
36+
cores = parts[2].split('/')[0].split()[0]
37+
protocol = extra.split('/')[1].split(' - ')[0] if '/' in extra else 'unknown'
38+
key = f"{cores}core-{protocol}"
39+
40+
if entry['name'] == 'logs_received_rate':
41+
throughput[key] = entry['value']
42+
elif entry['name'] == 'cpu_percentage_normalized_avg':
43+
cpu_norm[key] = entry['value']
44+
45+
# Calculate scaling metrics
46+
scaling_metrics = []
47+
for protocol in ['OTLP-ATTR-OTLP', 'OTAP-ATTR-OTLP']:
48+
baseline = throughput.get(f"1core-{protocol}", 1)
49+
50+
for cores in ['1', '2', '4', '8']:
51+
key = f"{cores}core-{protocol}"
52+
if key in throughput:
53+
cores_int = int(cores)
54+
actual_speedup = throughput[key] / baseline if baseline > 0 else 0
55+
ideal_speedup = cores_int
56+
efficiency = (actual_speedup / ideal_speedup) * 100 if ideal_speedup > 0 else 0
57+
per_core = throughput[key] / cores_int
58+
59+
# Add efficiency metric (higher is better)
60+
scaling_metrics.append({
61+
"name": "scaling_efficiency",
62+
"unit": "%",
63+
"value": efficiency,
64+
"extra": f"Continuous - Saturation - {cores} Core(s)/{protocol} - Scaling Efficiency"
65+
})
66+
67+
# Add per-core throughput (should remain constant for linear scaling)
68+
scaling_metrics.append({
69+
"name": "per_core_throughput",
70+
"unit": "logs/sec/core",
71+
"value": per_core,
72+
"extra": f"Continuous - Saturation - {cores} Core(s)/{protocol} - Per-Core Throughput"
73+
})
74+
75+
# Add speedup metric
76+
scaling_metrics.append({
77+
"name": "speedup",
78+
"unit": "x",
79+
"value": actual_speedup,
80+
"extra": f"Continuous - Saturation - {cores} Core(s)/{protocol} - Speedup vs 1-core"
81+
})
82+
83+
# Merge with original data
84+
data.extend(scaling_metrics)
85+
86+
with open(output_file, 'w') as f:
87+
json.dump(data, f, indent=2)
88+
89+
print(f"Added {len(scaling_metrics)} scaling metrics")
90+
print(f"Output written to: {output_file}")
91+
92+
93+
if __name__ == '__main__':
94+
if len(sys.argv) != 3:
95+
print(__doc__)
96+
sys.exit(1)
97+
98+
input_file = sys.argv[1]
99+
output_file = sys.argv[2]
100+
101+
compute_scaling_metrics(input_file, output_file)

‎.github/workflows/scripts/generate-scaling-summary.py‎

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
#!/usr/bin/env python3
2+
"""
3+
Generate scaling analysis summary table from benchmark data.
4+
5+
Reads benchmark JSON with computed metrics and generates a markdown summary table
6+
showing throughput, speedup, efficiency, CPU usage, and dropped logs for each configuration.
7+
8+
Usage:
9+
python generate-scaling-summary.py <input-file> [output-file]
10+
11+
Examples:
12+
# Print to console
13+
python generate-scaling-summary.py output-saturation.json
14+
15+
# Write to file
16+
python generate-scaling-summary.py output-saturation.json summary.md
17+
18+
# Append to GitHub Step Summary (CI mode)
19+
GITHUB_STEP_SUMMARY=summary.md python generate-scaling-summary.py output-saturation.json
20+
"""
21+
22+
import json
23+
import sys
24+
import os
25+
26+
27+
def generate_summary(input_file, output_file=None):
28+
"""Generate scaling analysis summary table."""
29+
30+
with open(input_file) as f:
31+
data = json.load(f)
32+
33+
# Group metrics by configuration
34+
metrics = {}
35+
for entry in data:
36+
extra = entry['extra']
37+
parts = extra.split(' - ')
38+
if len(parts) < 3:
39+
continue
40+
41+
cores = parts[2].split('/')[0].split()[0]
42+
protocol = extra.split('/')[1].split(' - ')[0] if '/' in extra else 'unknown'
43+
key = f"{cores}core-{protocol}"
44+
45+
if key not in metrics:
46+
metrics[key] = {}
47+
48+
name = entry['name']
49+
if name == 'logs_received_rate':
50+
metrics[key]['throughput'] = entry['value']
51+
elif name == 'cpu_percentage_normalized_avg':
52+
metrics[key]['cpu'] = entry['value']
53+
elif name == 'speedup':
54+
metrics[key]['speedup'] = entry['value']
55+
elif name == 'scaling_efficiency':
56+
metrics[key]['efficiency'] = entry['value']
57+
elif name == 'dropped_logs_percentage':
58+
metrics[key]['dropped'] = entry['value']
59+
60+
# Build markdown output
61+
lines = []
62+
lines.append("\n## 🚀 Core Scaling Analysis\n")
63+
lines.append("### OTLP Protocol\n")
64+
lines.append("| Cores | Throughput (logs/s) | Speedup | Efficiency | CPU % | Dropped % |\n")
65+
lines.append("|-------|--------------------:|--------:|-----------:|------:|----------:|\n")
66+
67+
for cores in ['1', '2', '4', '8']:
68+
key = f"{cores}core-OTLP-ATTR-OTLP"
69+
if key in metrics and 'throughput' in metrics[key]:
70+
m = metrics[key]
71+
speedup = m.get('speedup', 0)
72+
efficiency = m.get('efficiency', 0)
73+
cpu = m.get('cpu', 0)
74+
throughput = m.get('throughput', 0)
75+
dropped = m.get('dropped', 0)
76+
77+
# Add status emoji
78+
eff_emoji = "🟢" if efficiency >= 80 else "🟡" if efficiency >= 60 else "🔴"
79+
cpu_emoji = "✅" if cpu >= 90 else "⚠️" if cpu >= 70 else "❌"
80+
81+
lines.append(f"| {cores} {eff_emoji} | {throughput:>15,.0f} | {speedup:>5.2f}x | {efficiency:>8.1f}% | {cpu:>4.1f}% {cpu_emoji} | {dropped:>6.2f}% |\n")
82+
83+
lines.append("\n### OTAP Protocol\n")
84+
lines.append("| Cores | Throughput (logs/s) | Speedup | Efficiency | CPU % | Dropped % |\n")
85+
lines.append("|-------|--------------------:|--------:|-----------:|------:|----------:|\n")
86+
87+
for cores in ['1', '2', '4', '8']:
88+
key = f"{cores}core-OTAP-ATTR-OTLP"
89+
if key in metrics and 'throughput' in metrics[key]:
90+
m = metrics[key]
91+
speedup = m.get('speedup', 0)
92+
efficiency = m.get('efficiency', 0)
93+
cpu = m.get('cpu', 0)
94+
throughput = m.get('throughput', 0)
95+
dropped = m.get('dropped', 0)
96+
97+
# Add status emoji
98+
eff_emoji = "🟢" if efficiency >= 80 else "🟡" if efficiency >= 60 else "🔴"
99+
cpu_emoji = "✅" if cpu >= 90 else "⚠️" if cpu >= 70 else "❌"
100+
101+
lines.append(f"| {cores} {eff_emoji} | {throughput:>15,.0f} | {speedup:>5.2f}x | {efficiency:>8.1f}% | {cpu:>4.1f}% {cpu_emoji} | {dropped:>6.2f}% |\n")
102+
103+
lines.append("\n**Legend:**\n")
104+
lines.append("- 🟢 Efficiency ≥80% | 🟡 60-80% | 🔴 <60%\n")
105+
lines.append("- ✅ CPU ≥90% (saturated) | ⚠️ 70-90% | ❌ <70% (under-utilized)\n")
106+
lines.append("- **Ideal Linear Scaling:** Efficiency = 100%, Speedup = # of cores\n\n")
107+
108+
markdown_output = ''.join(lines)
109+
110+
# Determine output destination
111+
# Priority: 1) explicit output_file, 2) GITHUB_STEP_SUMMARY env var, 3) stdout
112+
if output_file:
113+
with open(output_file, 'a') as f:
114+
f.write(markdown_output)
115+
print(f"Summary appended to: {output_file}")
116+
elif 'GITHUB_STEP_SUMMARY' in os.environ:
117+
summary_file = os.environ['GITHUB_STEP_SUMMARY']
118+
with open(summary_file, 'a') as f:
119+
f.write(markdown_output)
120+
print(f"Summary appended to GitHub Step Summary: {summary_file}")
121+
else:
122+
print(markdown_output)
123+
124+
125+
if __name__ == '__main__':
126+
if len(sys.argv) < 2:
127+
print(__doc__)
128+
sys.exit(1)
129+
130+
input_file = sys.argv[1]
131+
output_file = sys.argv[2] if len(sys.argv) > 2 else None
132+
133+
generate_summary(input_file, output_file)

0 commit comments

Comments
 (0)