latency-numbers-every-programmer-should-know/chart.html at main · rivet-dev/latency-numbers-every-programmer-should-know · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
<!DOCTYPE html>
<html>
<head>
<title>AI Latency Benchmarks</title>
<script src="https://cdn.plot.ly/plotly-2.35.0.min.js"></script>
<style>
  body { font-family: -apple-system, sans-serif; margin: 0; padding: 20px; background: #fafafa; }
  h1 { font-size: 22px; margin-bottom: 2px; }
  .subtitle { color: #666; font-size: 13px; margin: 0 0 24px; }
  .chart { background: #fff; border-radius: 12px; border: 1px solid #e2e8f0; padding: 16px; margin-bottom: 24px; }
</style>
</head>
<body>

<h1>AI Latency Benchmarks</h1>
<p class="subtitle">AWS Lambda (us-east-1, 3072 MB) — 100 samples across 10 fresh sandboxes</p>

<div id="sandbox" class="chart"></div>
<div id="provider" class="chart"></div>

<script>
function parseCSV(text) {
  const [header, ...rows] = text.trim().split('\n');
  const cols = header.split(',').slice(2);
  const data = {};
  cols.forEach(c => data[c] = []);
  rows.forEach(row => {
    const vals = row.split(',');
    cols.forEach((c, i) => {
      const v = parseFloat(vals[i + 2]);
      if (!isNaN(v)) data[c].push(v);
    });
  });
  return data;
}

function box(y, name, x, color) {
  return {
    y, name,
    x: y.map(() => x),
    type: 'box',
    marker: { color },
    boxmean: true,
    boxpoints: 'outliers',
    width: 0.35,
  };
}

const sb = parseCSV(`iteration,sample,e2b:coldstart,e2b:native-exec,e2b:agent-exec,e2b:agent-health,daytona:coldstart,daytona:native-exec,daytona:agent-exec,daytona:agent-health
1,1,671.84,281.01,86.51,65.1,546.14,30.56,16.73,9.21
1,2,477.87,88.72,78.7,73.45,884.22,12.22,17.44,8.06
1,3,454.4,79.13,65.74,73.73,741.44,25.05,17.71,8.46
1,4,269.78,79.64,68.01,75.48,597.78,11.38,12.05,8.59
1,5,257.13,78.43,73.5,74.49,752.4,22.7,9.96,8.95
1,6,264.77,70.15,66.86,73.13,528.58,23.37,9.36,9.42
1,7,259.08,80.69,65.07,65.19,762.61,22.59,10.22,8.24
1,8,284.71,79.52,74.88,73.14,912.21,23.18,9.67,8.48
1,9,267.49,81.55,74.34,64.15,537.17,23.47,8.84,7.84
1,10,259.03,79.94,66.1,74.31,986.03,11.85,10.52,7.15
2,1,274.56,90.82,88.97,64.38,534.96,25.88,17.3,25.84
2,2,256.63,93.03,77.65,64.52,986.69,20.71,33.49,24.57
2,3,391.69,72.07,66.21,73.42,856.63,16.07,27.17,24.06
2,4,255.19,80.11,68.05,75.08,511.44,18.81,26.62,27.07
2,5,419.51,81.97,74.43,63.96,539.74,16.76,46.54,24.73
2,6,264.87,71.38,74.96,64.72,529.89,17.96,26.2,26.27
2,7,272.54,81.15,65.83,65.17,1114.22,19.2,28.37,24.48
2,8,244.47,72.79,67.03,65.27,534.79,19.23,27.06,27.23
2,9,252.5,78.91,65.38,72.76,547.41,19.72,26.96,9.92
2,10,258.29,73.15,67.47,65.98,875.37,19.66,26.44,24.3
3,1,338.67,82.61,79.35,64.88,1650.48,99.97,91.37,83.22
3,2,244.43,92.4,85.15,72.38,982.65,106.33,36.97,31.69
3,3,246.67,77.86,233.23,64.67,639.6,114.52,31.72,82.06
3,4,268.2,81.27,74.32,230.78,651.56,94.84,98.32,28.64
3,5,251.93,396.4,66.61,66.85,420.14,86.02,98.81,27.97
3,6,261.59,81.06,66.54,66.73,421.02,88.69,34.3,29.16
3,7,250.08,227.44,66.68,64.77,1541.35,107.37,32.62,83.45
3,8,255.71,72.51,65.58,73.5,651.79,97.1,32.23,29.29
3,9,250.78,91.02,66.74,67.67,635.93,130.13,30.98,87.44
3,10,246.49,80.24,65.7,64.46,1282.58,95.91,32.38,29.64
4,1,275.4,82.48,78.45,66.35,1448.41,33.11,26.25,24.08
4,2,434.52,83.59,241.19,73.9,880.74,25.92,15.32,9.97
4,3,347.54,78.82,68.99,73.27,757.69,30.76,9.94,22.87
4,4,381.05,81.08,74.24,234.09,762.68,28.3,24.65,7.71
4,5,263.09,79.1,66.6,65.27,872.74,28.18,10.35,19.89
4,6,261.92,80.09,67.18,72.75,523.71,26.47,9.93,8.98
4,7,286.95,72.05,67.52,77.89,537.87,27.58,22.63,25.34
4,8,245.45,71.55,78.98,64.82,546.51,26.76,8.47,9.46
4,9,250.23,71.49,76.07,73.59,542.77,27.15,11.53,10.72
4,10,270.3,73.2,69.35,65.02,766.23,22.52,10.56,9.09
5,1,279.71,84.93,80.03,63.59,875.66,49.47,14.42,7.95
5,2,255.98,82.07,79.1,64.85,530.12,11.86,14.85,7.36
5,3,388.91,72.52,66.3,64.71,890.63,32.19,9.98,8.64
5,4,315.08,119.63,75.79,73.8,544.06,11.57,9.87,8.42
5,5,259.98,79.3,70.45,73.61,544.94,12.13,21.07,6.83
5,6,272.26,71.14,67.91,73.01,528.88,11.39,8.93,6.78
5,7,312.83,79.7,75.84,65.59,882.25,10.66,9.46,9.14
5,8,361.36,78.89,67.42,64.49,779.99,20.63,21.66,7.77
5,9,264.42,72.25,66.18,66.21,542.93,25.53,8.73,21.61
5,10,253.7,70.37,75.29,65.23,426.83,24.65,9.12,25.58
6,1,288.44,91.46,79.91,65.73,881.14,17.62,26.94,12.92
6,2,251.9,92.12,77.31,65,653.3,10.39,14.83,9.32
6,3,253.32,79.05,266.02,64.63,534.54,23.29,9.58,8.65
6,4,429.7,79.07,66.16,65.66,419.15,24.61,8.94,8.87
6,5,255.09,79.69,66.54,64.33,771.85,23.83,20.36,8.95
6,6,243.26,71.92,66.94,65.31,534.54,10.95,20.08,8.2
6,7,242.95,79.82,67.65,64.96,774.43,11.79,12.8,11.36
6,8,252.35,81.09,67.13,65.03,549.74,10.92,10.22,7.7
6,9,248.42,74.04,89.78,64.19,533.38,10.91,8.42,8.28
6,10,257.08,79.02,66.67,64.34,532.7,11.22,20.6,23.89
7,1,286.23,86.44,86.02,73.35,1122.82,221.71,222.03,214.56
7,2,261.66,85.73,76.14,66.25,551.03,218.92,223.02,71.89
7,3,248.99,72.2,67.24,73.33,652.49,78.15,74.15,211.45
7,4,246.17,72.86,73.92,75.6,1110.99,77.16,212.57,73.28
7,5,258.92,274.47,75.69,78.2,906.56,81.93,216.78,213.06
7,6,283.35,74.11,67.95,64.57,773.81,81.25,215.28,73.35
7,7,262.25,71.04,77.1,65.43,767.35,78.76,218.31,211.74
7,8,254.5,80.03,74.66,73.39,1325.99,80.29,213.07,72.43
7,9,251.18,81.37,67.56,64.69,549.26,79.39,215.96,215.16
7,10,260.11,71.09,75.29,64.79,1537.75,79.85,214.43,72.75
8,1,633.77,82.81,77.42,65.15,865.18,38.39,17.47,9.64
8,2,257.04,81.66,75.65,64.21,650.22,28.24,18.55,8.53
8,3,258.06,81.97,68.45,65.01,546.69,28.63,10.89,10.34
8,4,278.79,73.18,67.3,65.31,537.25,25.73,9.91,8.75
8,5,257.6,74.15,66.51,73.1,537.92,34.5,10.25,8.31
8,6,725.64,81.17,67.51,64.9,656.71,28.69,22.14,8.14
8,7,256.18,75.6,74.48,65.88,645.95,13.67,25.5,9.6
8,8,254.04,72.3,66.21,63.58,655.24,30.87,10.73,8.58
8,9,265.91,81.31,69.42,69.45,1987.43,27.06,51.06,9.31
8,10,253.51,72.5,67.73,66.95,422.32,27.12,11.03,8.57
9,1,484.65,79.98,80.08,235.75,1558.69,16.41,26.76,8.73
9,2,488.27,80.38,94.25,102.66,778.2,11.06,32.5,7.78
9,3,264.34,71.55,75.81,65.71,437.54,11.73,8.9,18.66
9,4,259.2,79.97,85.16,113.12,974.92,11.13,8.64,12.92
9,5,262.17,70.12,79.63,63.67,519.78,11.69,8.97,22.46
9,6,277.7,79.17,90.79,89.03,537.32,11.63,8.75,7.98
9,7,279.96,80.59,68.79,75.75,879.29,11.47,18.59,20.69
9,8,270.02,71.75,88.5,81.24,872.98,14.54,21.18,8.54
9,9,261.8,78.62,68.88,64.9,674.08,11.23,9.81,8.38
9,10,281.32,78.19,89.13,76.92,531.41,24.82,21,6.85
10,1,282.52,88.64,84.93,71.9,538.18,2104.77,36.31,30.18
10,2,262.25,89.29,80.09,65.83,781.62,88.02,37.89,29.87
10,3,267.79,70.34,65.86,75.89,1340.12,35.78,30.28,30.16
10,4,287.05,69.98,67.11,73.45,540.91,84.03,31.94,27.81
10,5,266.82,77.99,75.94,74.77,536.75,88.36,29.47,29.56
10,6,414.26,70.99,71.18,75.88,758.86,88.37,90.17,82.95
10,7,265.73,69.14,67,77.05,545.53,88.32,31.76,28.36
10,8,260.76,71.2,67.98,64.72,646.03,90.41,85.52,83.37
10,9,259.91,73.73,69.24,66.49,527.37,36.52,35.36,81.39
10,10,469.55,89.29,67.01,64.31,1051.3,90.14,85.99,29.27
`);
const pr = parseCSV(`iteration,sample,llm:anthropic-haiku,llm:anthropic-opus,llm:openai-mini,llm:openai,llm:openrouter-haiku,llm:openrouter-opus,llm:openrouter-openai-mini,llm:openrouter-openai
1,1,404.49,1526.61,1024.9,847.86,1455.89,2730.68,3565.56,6664.64
1,2,610.71,1587.24,1141.9,991.28,1286.91,5677.07,1053.84,1198.42
1,3,385.97,1754.47,748.44,747.03,1862.92,1724.18,1197.61,2325.25
1,4,454.05,1513.77,1918.71,872.87,1009.2,1356.3,1783.87,1071.63
1,5,1051.1,1422,680.81,735.73,1531.38,4343.03,2459.6,1130.09
1,6,388.07,1681.97,761.38,1051.17,1306.06,1458.28,4248.12,1878.1
1,7,396.18,1721.64,981.23,671.78,1897.65,3571.59,1155.79,1248.26
1,8,959.34,1495.94,1118.66,760.9,1012.17,1469.74,1280.72,1166.76
1,9,869.93,1352.94,708.91,815.31,944.96,2212.44,1154.2,3409.24
1,10,483.53,1709.98,479.42,698.43,1312.28,2002.32,1175.72,2479.44
2,1,663.88,1475.76,751.05,625.18,1137.87,2350.08,989.38,1636.65
2,2,600.66,1480.97,852.46,616.63,1437.19,6571.5,1418.53,3702.37
2,3,567.17,1641.86,659.23,803.15,1530.38,1677.74,2018.78,3527.42
2,4,428.02,1718.15,1077.42,617.61,1074.29,1904.02,1107.59,1204.06
2,5,5146.84,1797.96,745.35,735.68,1132.35,1964.79,4071.45,3596.58
2,6,420.97,1582.12,691.07,774.87,961.53,1572.16,3852.31,1544.92
2,7,1208.71,1470.7,567.77,791.92,1148.27,2182.81,4548.91,1880.01
2,8,388.81,1671.59,568.99,842.06,959.56,1390.33,2738.76,2034.83
2,9,348.89,1476.79,828.25,743.08,1247.19,4560.75,3871.63,5081.73
2,10,404.24,1835.94,519.4,811.62,1084.96,3681.31,1760.32,1548.33
3,1,523.16,1446.82,838.03,809.23,1318.44,1371.68,907.8,4529.98
3,2,2465.02,2056.41,751.59,726.44,1233.2,7360.56,1119.13,3468.56
3,3,557.88,2326.4,713.19,857.39,1523.51,7215.3,1425.4,2761.65
3,4,509.72,2012.52,704.85,877.99,949.91,1601.96,1965.74,999.07
3,5,2432.79,1400.37,794.99,796.94,1018.09,1792.87,934.42,3931.7
3,6,447.88,2072,831.86,840.61,2154.81,1556.36,2692.52,1340.52
3,7,415.41,1512.45,626.04,948.71,1285.55,1583.44,2704.65,4000.12
3,8,1059.39,1438.93,528.15,792.66,1205.54,1919.79,1435.64,5458.8
3,9,460.25,1490.29,735.02,1180.02,2200.05,5173.3,2154.82,1123.61
3,10,477.78,1585.92,588.92,980.58,1052.13,3025.73,1022.57,3739.67
4,1,543.66,1480.03,645.01,894.32,2402.14,1434.15,862.37,3238.46
4,2,355.13,1513.97,796.08,804.19,1418.72,1645.33,1111.86,1159.29
4,3,448.45,1698.59,669.54,855.56,1226.31,1379.55,2259.94,3849.6
4,4,1039.14,1413.72,526.08,815.76,1001.38,3213.8,1335.34,2731.77
4,5,406.34,1892.8,642.61,733.76,1097.57,1538.6,1095.47,2788.99
4,6,474.28,1647.77,652.96,845.34,1503.75,10867.64,4164.05,1553.25
4,7,430.79,1388.3,629.12,1082.21,1526.07,1812.11,1012,3203.21
4,8,834.66,1381.99,674.03,867.99,1201.61,1455.06,1036.89,3687.02
4,9,372.02,1613.67,503.94,842.27,1484.78,2924.19,4085.37,2624.76
4,10,472.04,1420.41,742.83,800.8,839.47,1691.27,1282.42,1080.28
5,1,345.71,1325.77,601.55,860.21,997.2,3240.85,976.35,838.06
5,2,393.08,1610.98,602.58,1298.48,1349.64,2256.53,1319.07,1766.8
5,3,756.81,2148.09,502.65,760,1386.75,1521.11,3276.81,3301.91
5,4,457.05,1486.24,716.05,611.58,993.32,3516,2227.6,3331.02
5,5,520.04,1575.92,973.95,768.44,2022.12,3046.07,2140.79,4170.89
5,6,647.74,1853.17,731.64,679.9,1408.42,5488.28,2655.83,3094
5,7,481.77,1789.36,698.13,590.01,1524.77,1973.79,1025.95,3446.98
5,8,627.8,1695.41,573.05,783.04,1207.19,4635.07,1045.86,2559.41
5,9,696.78,1623.37,760.27,740.56,936.71,1751.19,1475.29,1493.43
5,10,532.52,1583.09,618.64,907.3,1618.63,1842.38,4443.98,2242.19
6,1,369.16,1395.96,850.1,864.69,2616.46,1787.79,3642.78,1476.87
6,2,364.69,1504.33,644.38,657.26,1365.34,2132.83,1395.89,3286.46
6,3,381.28,1653.28,610.35,810.32,1318.97,12110.34,943.44,1662.1
6,4,487.16,1595.24,775.1,778.95,1146.84,1677.15,1151.7,3746.99
6,5,720.29,2109.46,566.8,1078.96,1204.14,2309.48,1129.61,3072.11
6,6,496.76,2016.68,723.76,1053,1089.12,3694.03,1151.45,1955.91
6,7,702.32,1653.11,651.54,733.66,989.58,1733.9,2692.25,969.21
6,8,431.02,1558.25,760.53,840.04,1087.44,4529.39,3969.71,1675.39
6,9,825.92,1630.61,783.76,817.8,904.13,2832.63,4299.91,2329.85
6,10,627.78,1879.47,656.97,788.12,1843.72,1990.87,2351.23,1312.14
7,1,431.32,1588.3,870.85,630.93,995.45,2076.75,3012.28,1350.73
7,2,410.98,1500.42,626,811.1,875.48,6508.15,1007.79,2642.2
7,3,453.18,1371.92,706.82,1023.2,1482.49,2895.85,959.47,1151.97
7,4,546.2,1641.33,602.79,846.31,1715.58,1542.7,1873.88,2652.48
7,5,665.48,1573.67,654.58,719.34,3045.13,1461.86,3488.94,1258.96
7,6,402.07,1287.27,2339.19,1043.46,1056.59,3255.63,1125.02,1252.57
7,7,453.23,1517.93,677.62,1213.45,1801.33,1663.43,1031.81,3285.39
7,8,502.03,1393.92,651.83,860.9,1167.83,1505.83,1190.3,4293.58
7,9,513.84,1536.52,666.71,802.61,1028.44,1638.9,3775.75,1133.24
7,10,486.49,1432.65,568.04,835.17,1007.55,5021.88,1795.9,1089.06
8,1,455.88,1409.23,872.71,1063.5,1581.5,1694.17,1158.66,962.68
8,2,400.3,1402.51,740.45,814.93,932.05,5800.38,1041.14,2195.08
8,3,462.49,2236.95,883.67,809.56,1270.72,6896.32,1190.32,3466.41
8,4,461.44,1670.69,605.71,768.01,1929.45,1931.85,967.82,3514.01
8,5,376.49,1391.63,767.02,888.32,1140.38,2794.98,894.8,3751.38
8,6,393.24,1513.26,644.19,809.25,806.97,3497.36,1066.34,2749.24
8,7,532.82,1430.79,596.56,748.72,959.45,2007.7,4434.62,3707.91
8,8,527.39,1661.28,511.17,775.24,1301.46,2088.12,1187.47,2293.65
8,9,432.73,1510.28,610.16,1022.29,1216.02,1818.5,1039.51,2904.7
8,10,439.06,1493.7,646.77,711.52,1189.4,1768.57,2123.53,783.48
9,1,403.09,1388.9,707.55,701.58,1092.47,1567.88,1812.46,3605.21
9,2,608.29,1625.18,704.84,905.2,1056.28,4009.22,2299.78,1965.66
9,3,392.96,1466.18,987.17,886.86,1481.99,5168.56,3632.91,1846.02
9,4,436.85,1910.09,701.65,817.56,1046.65,1529.57,2349.88,3848.9
9,5,881.21,1742.66,798.62,722.52,1854.84,1506.05,920.24,1271.43
9,6,460.28,1422.43,814.01,765.07,1294.35,1974.41,958.26,3620.38
9,7,451.44,1672.04,679.44,723.54,2346.72,5140.81,3420.14,3738.56
9,8,836.28,1557.17,704.77,988.37,2239.26,4892.54,1417.87,1156.85
9,9,462.82,2031.9,741.39,1069.72,859.19,1819.59,1155.73,3470.02
9,10,532.95,1585.51,609.32,741.89,2343.76,2690.68,1205.93,1077.32
10,1,405.52,1412.59,735.09,856.67,1041.68,2581.4,2519.32,1084.66
10,2,443.41,1507.49,705.8,1082.18,1074.27,2033.95,1043.8,2072.76
10,3,514.28,1408.94,553.48,813.73,1951.61,1444.31,1016.95,4477.94
10,4,395.04,1600.23,1355.57,922.6,1383.65,4866.91,2968.45,1545.98
10,5,869.13,1526.96,727.64,840.01,1368.87,4830.2,1892.84,2209.39
10,6,429.56,1567.35,641.86,885.86,1030.51,1418.49,4996.12,1353.02
10,7,419.35,1735.7,980.85,910.36,1154.86,2044.68,1130.48,772.53
10,8,460.36,1488.72,668.02,938.53,1509.81,1581.3,871.12,4258.68
10,9,503.61,1654.48,686.27,821.44,1456.51,4822.92,3330.94,2765.64
10,10,609.36,1455.51,656.69,760.66,924.03,7670.68,1201.99,965.24
`);

const E2B = '#3b82f6';
const DAY = '#10b981';
const DIR = '#8b5cf6';
const OR  = '#f59e0b';

Plotly.newPlot('sandbox', [
  box(sb['e2b:coldstart'], 'E2B', 'Coldstart', E2B),
  box(sb['daytona:coldstart'], 'Daytona', 'Coldstart', DAY),
  box(sb['e2b:native-exec'], 'E2B', 'Tool Execution', E2B),
  box(sb['daytona:native-exec'], 'Daytona', 'Tool Execution', DAY),
  box(sb['e2b:agent-health'], 'E2B', 'Network Round Trip', E2B),
  box(sb['daytona:agent-health'], 'Daytona', 'Network Round Trip', DAY),
].map((t, i) => ({ ...t, legendgroup: t.name, showlegend: i < 2 })), {
  title: { text: 'Sandbox Provider Latency', font: { size: 18 } },
  yaxis: { title: 'ms', rangemode: 'tozero' },
  boxmode: 'group',
  height: 500,
  legend: { orientation: 'h', y: -0.12 },
  margin: { t: 50, b: 80 },
}, { responsive: true });

Plotly.newPlot('provider', [
  box(pr['llm:anthropic-haiku'], 'Direct', 'Haiku 4.5', DIR),
  box(pr['llm:openrouter-haiku'], 'OpenRouter', 'Haiku 4.5', OR),
  box(pr['llm:anthropic-opus'], 'Direct', 'Opus 4.6', DIR),
  box(pr['llm:openrouter-opus'], 'OpenRouter', 'Opus 4.6', OR),
  box(pr['llm:openai-mini'], 'Direct', 'GPT-5 mini', DIR),
  box(pr['llm:openrouter-openai-mini'], 'OpenRouter', 'GPT-5 mini', OR),
  box(pr['llm:openai'], 'Direct', 'GPT-5', DIR),
  box(pr['llm:openrouter-openai'], 'OpenRouter', 'GPT-5', OR),
].map((t, i) => ({ ...t, legendgroup: t.name, showlegend: i < 2 })), {
  title: { text: 'LLM Inference Latency', font: { size: 18 } },
  yaxis: { title: 'ms', rangemode: 'tozero' },
  boxmode: 'group',
  height: 500,
  legend: { orientation: 'h', y: -0.12 },
  margin: { t: 50, b: 80 },
}, { responsive: true });
</script>
</body>
</html>