Skip to content

Commit 5fbb832

Browse files
author
shengtenghou4-star
committed
feat: Add API benchmark suite for #30
- Created benchmarks/ directory with run.js - Benchmark tool: autocannon (Node.js) - Measures: p50, p95, p99 latency, RPS, error rate, TTFB - Outputs JSON and Markdown results - CI workflow with threshold checks Metrics captured: - Latency percentiles (p50, p95, p99) - Requests per second - Error rate percentage - Time to first byte CI threshold check will fail if: - p99 latency > 1000ms - p95 latency > 500ms - Error rate > 1% - RPS < 100 AI agent: Claude Code 3.7 (Sonnet) Model: claude-sonnet-4-20250514 Mode: Automatic execution
1 parent 688d915 commit 5fbb832

5 files changed

Lines changed: 375 additions & 1 deletion

File tree

‎.github/workflows/benchmark.yml‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
name: API Benchmark
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
benchmark:
11+
runs-on: ubuntu-latest
12+
13+
steps:
14+
- uses: actions/checkout@v4
15+
16+
- name: Setup Node.js
17+
uses: actions/setup-node@v4
18+
with:
19+
node-version: '20'
20+
21+
- name: Install dependencies
22+
run: |
23+
npm install
24+
npm install -g autocannon
25+
26+
- name: Install API dependencies
27+
run: |
28+
cd apps/api
29+
npm install
30+
31+
- name: Start API server
32+
run: |
33+
cd apps/api
34+
npm run dev &
35+
sleep 5
36+
37+
- name: Run benchmarks
38+
run: |
39+
node benchmarks/run.js || true
40+
41+
- name: Check thresholds
42+
run: |
43+
node benchmarks/run.js
44+
45+
- name: Upload benchmark results
46+
if: always()
47+
uses: actions/upload-artifact@v4
48+
with:
49+
name: benchmark-results
50+
path: benchmarks/results/
51+
52+
- name: Comment on PR
53+
if: github.event_name == 'pull_request'
54+
uses: actions/github-script@v7
55+
with:
56+
script: |
57+
const fs = require('fs');
58+
const summary = fs.readFileSync('benchmarks/results/summary.md', 'utf8');
59+
github.rest.issues.createComment({
60+
issue_number: context.issue.number,
61+
owner: context.repo.owner,
62+
repo: context.repo.repo,
63+
body: '## Benchmark Results\n\n' + summary
64+
});

‎benchmarks/README.md‎

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,29 @@
1+
# Benchmarks
2+
3+
API performance benchmarking suite for FreelanceFlow.
4+
5+
## Setup
6+
7+
```bash
8+
# Install autocannon
9+
npm install -g autocannon
10+
11+
# Or use npx
12+
npx autocannon --help
13+
```
14+
15+
## Running Benchmarks
16+
17+
```bash
18+
# Run all benchmarks
19+
npm run benchmark
20+
21+
# Or directly
22+
node benchmarks/run.js
23+
```
24+
25+
## Output
26+
27+
- JSON results: `benchmarks/results/<endpoint>-<timestamp>.json`
28+
- Markdown summary: `benchmarks/results/summary.md`
29+
- Thresholds: `benchmarks/thresholds.json`

‎benchmarks/run.js‎

Lines changed: 273 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,273 @@
1+
#!/usr/bin/env node
2+
/**
3+
* FreelanceFlow API Benchmark Suite
4+
* Measures: p50, p95, p99 latency, RPS, error rate, TTFB
5+
*
6+
* Run: node benchmarks/run.js
7+
*/
8+
9+
import autocannon from 'autocannon';
10+
import { writeFileSync, mkdirSync, existsSync, readFileSync } from 'fs';
11+
import { join, dirname } from 'path';
12+
import { fileURLToPath } from 'url';
13+
14+
const __dirname = dirname(fileURLToPath(import.meta.url));
15+
const RESULTS_DIR = join(__dirname, 'results');
16+
17+
// Ensure results directory exists
18+
if (!existsSync(RESULTS_DIR)) {
19+
mkdirSync(RESULTS_DIR, { recursive: true });
20+
}
21+
22+
// API Base URL - can be overridden via environment
23+
const API_BASE = process.env.API_URL || 'http://localhost:3000/api';
24+
const AUTH_TOKEN = process.env.BENCHMARK_TOKEN || '';
25+
26+
// Endpoints to benchmark (all /api/* routes)
27+
const ENDPOINTS = [
28+
{ path: '/auth/register', method: 'POST', body: { email: 'bench@test.com', password: 'Test123!', name: 'Benchmark' } },
29+
{ path: '/auth/login', method: 'POST', body: { email: 'bench@test.com', password: 'Test123!' } },
30+
{ path: '/jobs', method: 'GET' },
31+
{ path: '/jobs', method: 'POST', body: { title: 'Test Job', budget: 5000 } },
32+
{ path: '/users', method: 'GET' },
33+
{ path: '/users', method: 'GET', pathSuffix: '/1' },
34+
{ path: '/proposals', method: 'GET' },
35+
{ path: '/proposals', method: 'POST', body: { jobId: 1, freelancerId: 1, amount: 500 } },
36+
{ path: '/reviews', method: 'GET' },
37+
{ path: '/search', method: 'GET', query: { q: 'developer' } },
38+
{ path: '/notifications', method: 'GET' },
39+
{ path: '/messages', method: 'GET' },
40+
];
41+
42+
// Benchmark settings
43+
const BENCHMARK_OPTIONS = {
44+
connections: 10,
45+
duration: 10,
46+
pipelining: 1,
47+
workers: 2,
48+
};
49+
50+
// Load thresholds
51+
let thresholds = { p50: 100, p95: 500, p99: 1000, errorRate: 1, rps: 100 };
52+
try {
53+
const thresholdFile = join(__dirname, 'thresholds.json');
54+
if (existsSync(thresholdFile)) {
55+
thresholds = JSON.parse(readFileSync(thresholdFile, 'utf8'));
56+
}
57+
} catch (e) {
58+
console.warn('Could not load thresholds, using defaults');
59+
}
60+
61+
/**
62+
* Run benchmark for a single endpoint
63+
*/
64+
async function benchmarkEndpoint(endpoint) {
65+
const url = `${API_BASE}${endpoint.path}${endpoint.pathSuffix || ''}${endpoint.query ? '?' + new URLSearchParams(endpoint.query).toString() : ''}`;
66+
67+
const options = {
68+
url,
69+
method: endpoint.method || 'GET',
70+
body: endpoint.body ? JSON.stringify(endpoint.body) : undefined,
71+
headers: {
72+
'Content-Type': 'application/json',
73+
...(AUTH_TOKEN ? { 'Authorization': `Bearer ${AUTH_TOKEN}` } : {}),
74+
},
75+
...BENCHMARK_OPTIONS,
76+
};
77+
78+
try {
79+
const result = await autocannon(options);
80+
return {
81+
endpoint: endpoint.path,
82+
method: endpoint.method,
83+
...extractMetrics(result),
84+
timestamp: new Date().toISOString(),
85+
};
86+
} catch (error) {
87+
return {
88+
endpoint: endpoint.path,
89+
method: endpoint.method,
90+
error: error.message,
91+
timestamp: new Date().toISOString(),
92+
};
93+
}
94+
}
95+
96+
/**
97+
* Extract key metrics from autocannon result
98+
*/
99+
function extractMetrics(result) {
100+
const latencies = result.latency;
101+
102+
// Calculate percentiles from histogram
103+
const sorted = [...latencies].sort((a, b) => a - b);
104+
const p50 = sorted[Math.floor(sorted.length * 0.5)] || 0;
105+
const p95 = sorted[Math.floor(sorted.length * 0.95)] || 0;
106+
const p99 = sorted[Math.floor(sorted.length * 0.99)] || 0;
107+
108+
return {
109+
latency: {
110+
p50: Math.round(p50),
111+
p95: Math.round(p95),
112+
p99: Math.round(p99),
113+
mean: Math.round(result.latency.mean || 0),
114+
min: Math.round(result.latency.min || 0),
115+
max: Math.round(result.latency.max || 0),
116+
},
117+
rps: Math.round(result.requests.average || 0),
118+
throughput: {
119+
bytes: result.throughput.average || 0,
120+
mean: Math.round(result.throughput.mean || 0),
121+
},
122+
errors: result.errors || 0,
123+
timeouts: result.timeouts || 0,
124+
errorRate: result.errors ? ((result.errors / result.requests.total) * 100).toFixed(2) : 0,
125+
ttfb: {
126+
mean: Math.round(result.ttfb.mean || 0),
127+
min: Math.round(result.ttfb.min || 0),
128+
max: Math.round(result.ttfb.max || 0),
129+
},
130+
requests: {
131+
total: result.requests.total || 0,
132+
succeeded: (result.requests.total || 0) - (result.errors || 0) - (result.timeouts || 0),
133+
failed: result.errors || 0,
134+
},
135+
duration: result.duration,
136+
connections: result.connections,
137+
};
138+
}
139+
140+
/**
141+
* Check if results pass thresholds
142+
*/
143+
function checkThresholds(result) {
144+
const violations = [];
145+
146+
if (result.latency.p99 > thresholds.p99) {
147+
violations.push(`p99 latency (${result.latency.p99}ms) exceeds threshold (${thresholds.p99}ms)`);
148+
}
149+
if (result.latency.p95 > thresholds.p95) {
150+
violations.push(`p95 latency (${result.latency.p95}ms) exceeds threshold (${thresholds.p95}ms)`);
151+
}
152+
if (parseFloat(result.errorRate) > thresholds.errorRate) {
153+
violations.push(`Error rate (${result.errorRate}%) exceeds threshold (${thresholds.errorRate}%)`);
154+
}
155+
if (result.rps < thresholds.rps) {
156+
violations.push(`RPS (${result.rps}) below threshold (${thresholds.rps})`);
157+
}
158+
159+
return violations;
160+
}
161+
162+
/**
163+
* Generate markdown summary
164+
*/
165+
function generateMarkdown(results) {
166+
const timestamp = new Date().toISOString();
167+
const passCount = results.filter(r => !r.error && checkThresholds(r).length === 0).length;
168+
const failCount = results.length - passCount;
169+
170+
let md = '# FreelanceFlow API Benchmark Results\n\n';
171+
md += `**Generated:** ${timestamp}\n\n`;
172+
md += `**Summary:** ${passCount}/${results.length} endpoints passed thresholds\n\n`;
173+
md += '**Thresholds:**\n';
174+
md += `- p99 Latency: ${thresholds.p99}ms\n`;
175+
md += `- p95 Latency: ${thresholds.p95}ms\n`;
176+
md += `- Error Rate: ${thresholds.errorRate}%\n`;
177+
md += `- Min RPS: ${thresholds.rps}\n\n`;
178+
179+
md += '## Detailed Results\n\n';
180+
md += '| Endpoint | Method | p50 | p95 | p99 | RPS | Error Rate | Status |\n';
181+
md += '|----------|--------|-----|-----|-----|-----|-------------|--------|\n';
182+
183+
for (const result of results) {
184+
if (result.error) {
185+
md += `| ${result.endpoint} | ${result.method} | ERROR | ${result.error} | - | - | - |\n`;
186+
} else {
187+
const violations = checkThresholds(result);
188+
const status = violations.length === 0 ? 'PASS' : 'FAIL';
189+
md += `| ${result.endpoint} | ${result.method} | ${result.latency.p50}ms | ${result.latency.p95}ms | ${result.latency.p99}ms | ${result.rps} | ${result.errorRate}% | ${status} |\n`;
190+
}
191+
}
192+
193+
md += '\n## Failed Thresholds\n\n';
194+
for (const result of results) {
195+
if (!result.error) {
196+
const violations = checkThresholds(result);
197+
if (violations.length > 0) {
198+
md += `### ${result.endpoint} (${result.method})\n`;
199+
for (const v of violations) {
200+
md += `- ${v}\n`;
201+
}
202+
md += '\n';
203+
}
204+
}
205+
}
206+
207+
return md;
208+
}
209+
210+
/**
211+
* Main execution
212+
*/
213+
async function main() {
214+
console.log('Starting FreelanceFlow API Benchmark Suite\n');
215+
console.log(`Target: ${API_BASE}`);
216+
console.log(`Duration: ${BENCHMARK_OPTIONS.duration}s per endpoint`);
217+
console.log(`Connections: ${BENCHMARK_OPTIONS.connections}\n`);
218+
219+
const results = [];
220+
221+
for (const endpoint of ENDPOINTS) {
222+
console.log(`Benchmarking ${endpoint.method} ${endpoint.path}...`);
223+
const result = await benchmarkEndpoint(endpoint);
224+
results.push(result);
225+
226+
if (result.error) {
227+
console.log(` Error: ${result.error}`);
228+
} else {
229+
console.log(` p99: ${result.latency.p99}ms, RPS: ${result.rps}, Errors: ${result.errorRate}%`);
230+
}
231+
}
232+
233+
console.log('\nSaving results...');
234+
235+
// Save JSON results
236+
const timestamp = new Date().toISOString().replace(/[:.]/g, '-');
237+
const jsonFile = join(RESULTS_DIR, `benchmark-${timestamp}.json`);
238+
writeFileSync(jsonFile, JSON.stringify({
239+
timestamp,
240+
apiBase: API_BASE,
241+
thresholds,
242+
results,
243+
}, null, 2));
244+
console.log(` JSON: ${jsonFile}`);
245+
246+
// Save markdown summary
247+
const mdFile = join(RESULTS_DIR, 'summary.md');
248+
writeFileSync(mdFile, generateMarkdown(results));
249+
console.log(` Markdown: ${mdFile}`);
250+
251+
// Save individual endpoint results
252+
for (const result of results) {
253+
const safeName = result.endpoint.replace(/\//g, '-').replace(/^-/, '');
254+
const endpointFile = join(RESULTS_DIR, `${safeName}-${result.method.toLowerCase()}-${timestamp}.json`);
255+
writeFileSync(endpointFile, JSON.stringify(result, null, 2));
256+
}
257+
258+
// Check CI threshold
259+
const allPassed = results
260+
.filter(r => !r.error)
261+
.every(r => checkThresholds(r).length === 0);
262+
263+
console.log('\n' + '='.repeat(50));
264+
if (allPassed) {
265+
console.log('All endpoints passed threshold checks');
266+
process.exit(0);
267+
} else {
268+
console.log('Some endpoints failed threshold checks');
269+
process.exit(1);
270+
}
271+
}
272+
273+
main().catch(console.error);

‎benchmarks/thresholds.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"p50": 100,
3+
"p95": 500,
4+
"p99": 1000,
5+
"errorRate": 1,
6+
"rps": 100
7+
}

‎package.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
"scripts": {
99
"build": "echo \"Run package-specific builds (e.g. npm run build -w apps/web)\"",
1010
"lint": "echo \"No root lint configured\"",
11-
"test": "npm run test -w apps/api"
11+
"test": "npm run test -w apps/api",
12+
"benchmark": "node benchmarks/run.js"
1213
}
1314
}

0 commit comments

Comments
 (0)