-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathhow-ai-learns-from-feedback.html
More file actions
356 lines (321 loc) · 17.2 KB
/
Copy pathhow-ai-learns-from-feedback.html
File metadata and controls
356 lines (321 loc) · 17.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<title>Practice, Feedback, Repeat — how a chatbot learns to be good</title>
<link rel="preconnect" href="https://fonts.googleapis.com">
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
<link href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:opsz,wght@12..96,600;12..96,800&family=DM+Mono:wght@400;500&family=Work+Sans:wght@400;500&display=swap" rel="stylesheet">
<style>
:root{
--paper:#D9E0DC;
--pink:#FF3D8B;
--blue:#1B4CE0;
--yellow:#FFC61E;
--ink:#16181C;
--display:"Bricolage Grotesque","Arial Black",system-ui,sans-serif;
--body:"Work Sans",system-ui,sans-serif;
--mono:"DM Mono",ui-monospace,"Courier New",monospace;
--gut:clamp(20px,5vw,64px);
}
*{box-sizing:border-box}
html{scroll-behavior:smooth}
body{
margin:0;
background:var(--paper);
color:var(--ink);
font-family:var(--body);
-webkit-font-smoothing:antialiased;
overflow-x:hidden;
}
/* paper grain — the riso tell */
.grain{
position:fixed;inset:0;z-index:100;pointer-events:none;
opacity:.4;mix-blend-mode:multiply;
}
.wrap{max-width:1080px;margin:0 auto;padding:0 var(--gut)}
/* ---------- type ---------- */
.eyebrow{
font-family:var(--mono);font-size:12px;letter-spacing:.22em;
text-transform:uppercase;color:var(--blue);margin:0 0 18px;
}
h1{
font-family:var(--display);font-weight:800;
font-size:clamp(3.1rem,13vw,8.5rem);
line-height:.86;letter-spacing:-.035em;margin:0;
position:relative;
}
/* misregistration: the pink plate printed a hair off the black plate */
h1 .plate{position:relative;display:block}
h1 .plate::before{
content:attr(data-t);position:absolute;left:-.055em;top:.045em;
color:var(--pink);mix-blend-mode:multiply;z-index:-1;
}
h2{
font-family:var(--display);font-weight:800;
font-size:clamp(2rem,6.4vw,3.9rem);
line-height:.94;letter-spacing:-.03em;margin:0 0 20px;max-width:16ch;
}
p{font-size:clamp(1.05rem,2.4vw,1.35rem);line-height:1.45;margin:0;max-width:34ch}
.lede{font-family:var(--mono);font-size:clamp(.95rem,2.2vw,1.1rem);line-height:1.6;max-width:30ch}
.tag{
font-family:var(--mono);font-size:11px;letter-spacing:.18em;text-transform:uppercase;
display:inline-block;padding:6px 12px;border:1.5px solid var(--ink);border-radius:99px;
}
/* ---------- layout ---------- */
section{padding:clamp(70px,12vh,130px) 0;position:relative}
.rule{height:2px;background:var(--ink);opacity:.85}
.art{width:100%;height:auto;display:block}
.art-wide{max-width:820px;margin:0 auto}
.split{display:grid;grid-template-columns:1fr;gap:clamp(24px,4vw,56px);align-items:center}
@media(min-width:820px){
.split{grid-template-columns:1fr 1fr}
.split.flip .col-art{order:-1}
}
.duo{display:grid;grid-template-columns:1fr;gap:22px;margin-top:38px}
@media(min-width:700px){.duo{grid-template-columns:1fr 1fr;gap:32px}}
.card{border:2px solid var(--ink);border-radius:22px;padding:26px 24px 30px;background:transparent}
.card h3{font-family:var(--display);font-weight:800;font-size:1.7rem;margin:16px 0 8px;letter-spacing:-.02em}
.card p{font-size:1rem;max-width:30ch}
/* ink overprint */
.ink-pink{fill:var(--pink)}
.ink-blue{fill:var(--blue)}
.ink-yellow{fill:var(--yellow)}
.ink-dark{fill:var(--ink)}
svg .op{mix-blend-mode:multiply}
svg text{font-family:var(--display);font-weight:800;fill:var(--ink)}
svg .mono{font-family:var(--mono);font-weight:400}
.stroke{fill:none;stroke:var(--ink);stroke-width:5;stroke-linecap:round;stroke-linejoin:round}
.stroke-thin{fill:none;stroke:var(--ink);stroke-width:3.5;stroke-linecap:round}
/* hero */
.hero{padding-top:clamp(46px,9vh,90px);padding-bottom:clamp(40px,7vh,80px)}
.hero-art{max-width:520px;width:100%;margin-top:34px}
/* closer */
.closer{text-align:center;padding-bottom:clamp(90px,16vh,180px)}
.closer h2{max-width:none;margin-inline:auto;font-size:clamp(2.4rem,9vw,5.5rem)}
.closer p{margin-inline:auto;text-align:center}
/* reveal */
.rise{opacity:0;transform:translateY(26px);transition:opacity .7s ease,transform .7s cubic-bezier(.2,.7,.3,1)}
.rise.in{opacity:1;transform:none}
@media(prefers-reduced-motion:reduce){
.rise{opacity:1;transform:none;transition:none}
html{scroll-behavior:auto}
.spin{animation:none !important}
}
.spin{transform-origin:300px 300px;animation:turn 26s linear infinite}
@keyframes turn{to{transform:rotate(360deg)}}
:focus-visible{outline:3px solid var(--blue);outline-offset:3px}
</style>
</head>
<body>
<svg class="grain" xmlns="http://www.w3.org/2000/svg"><filter id="n"><feTurbulence type="fractalNoise" baseFrequency=".8" numOctaves="3"/><feColorMatrix type="saturate" values="0"/></filter><rect width="100%" height="100%" filter="url(#n)" opacity=".55"/></svg>
<!-- ================= HERO ================= -->
<header class="hero wrap">
<p class="eyebrow">A picture explainer</p>
<h1><span class="plate" data-t="Practice.">Practice.</span><span class="plate" data-t="Feedback.">Feedback.</span><span class="plate" data-t="Repeat.">Repeat.</span></h1>
<svg class="art hero-art" viewBox="0 0 520 300" role="img" aria-label="A speech bubble with a star inside, overlapping a blue circle">
<circle class="ink-blue op" cx="390" cy="150" r="112"/>
<path class="ink-pink op" d="M40 40h330a26 26 0 0 1 26 26v130a26 26 0 0 1-26 26H150l-72 62 12-62H40a26 26 0 0 1-26-26V66a26 26 0 0 1 26-26z"/>
<path class="ink-yellow op" d="M205 68l27 55 61 9-44 43 10 61-54-29-55 29 11-61-44-43 61-9z"/>
<path class="stroke" d="M40 40h330a26 26 0 0 1 26 26v130a26 26 0 0 1-26 26H150l-72 62 12-62H40a26 26 0 0 1-26-26V66a26 26 0 0 1 26-26z"/>
</svg>
<p class="lede" style="margin-top:34px">How a text-guessing machine gets turned into something worth talking to.</p>
</header>
<div class="wrap"><div class="rule"></div></div>
<!-- ================= 1. PRETRAINING ================= -->
<section class="wrap rise">
<div class="split">
<div>
<span class="tag">Where it starts</span>
<h2 style="margin-top:20px">It reads almost everything.</h2>
<p>And it learns exactly one trick: <em>guess the next word</em>.</p>
</div>
<div class="col-art">
<svg class="art" viewBox="0 0 480 320" role="img" aria-label="A stack of pages pouring into a box that guesses the next word">
<g class="op ink-yellow">
<rect x="20" y="196" width="150" height="26" rx="6"/>
<rect x="34" y="160" width="150" height="26" rx="6"/>
<rect x="12" y="124" width="150" height="26" rx="6"/>
<rect x="30" y="88" width="150" height="26" rx="6"/>
<rect x="20" y="52" width="150" height="26" rx="6"/>
</g>
<g class="stroke-thin">
<rect x="20" y="196" width="150" height="26" rx="6"/>
<rect x="34" y="160" width="150" height="26" rx="6"/>
<rect x="12" y="124" width="150" height="26" rx="6"/>
<rect x="30" y="88" width="150" height="26" rx="6"/>
<rect x="20" y="52" width="150" height="26" rx="6"/>
</g>
<path class="ink-pink op" d="M300 70h150a20 20 0 0 1 20 20v140a20 20 0 0 1-20 20H300a20 20 0 0 1-20-20V90a20 20 0 0 1 20-20z"/>
<path class="stroke" d="M300 70h150a20 20 0 0 1 20 20v140a20 20 0 0 1-20 20H300a20 20 0 0 1-20-20V90a20 20 0 0 1 20-20z"/>
<path class="stroke" d="M198 160h58"/>
<path class="ink-dark" d="M252 146l30 14-30 14z"/>
<text x="375" y="150" text-anchor="middle" font-size="30">the next</text>
<text x="375" y="186" text-anchor="middle" font-size="30">word is…</text>
<circle class="ink-blue op" cx="452" cy="72" r="18"/>
</svg>
</div>
</div>
</section>
<!-- ================= 2. THE PROBLEM ================= -->
<section class="wrap rise">
<div class="split flip">
<div>
<span class="tag">The problem</span>
<h2 style="margin-top:20px">Likely isn't the same as helpful.</h2>
<p>Both of these are things people write. Only one of them is any use to you.</p>
</div>
<div class="col-art">
<svg class="art" viewBox="0 0 480 340" role="img" aria-label="Two answer bubbles: a rambling one marked with a cross, a clear one marked with a tick">
<path class="ink-pink op" d="M24 20h300a22 22 0 0 1 22 22v78a22 22 0 0 1-22 22H120l-52 44 9-44H24A22 22 0 0 1 2 120V42a22 22 0 0 1 22-22z"/>
<path class="stroke-thin" d="M24 20h300a22 22 0 0 1 22 22v78a22 22 0 0 1-22 22H120l-52 44 9-44H24A22 22 0 0 1 2 120V42a22 22 0 0 1 22-22z"/>
<path class="stroke-thin" d="M34 56c14-12 28 12 42 0s28 12 42 0 28 12 42 0 28 12 42 0M34 96c14-12 28 12 42 0s28 12 42 0 28 12 42 0"/>
<g class="op ink-blue"><circle cx="404" cy="78" r="52"/></g>
<path class="stroke" d="M382 56l44 44M426 56l-44 44"/>
<path class="ink-yellow op" d="M24 196h300a22 22 0 0 1 22 22v78a22 22 0 0 1-22 22H120l-52 44 9-44H24a22 22 0 0 1-22-22v-78a22 22 0 0 1 22-22z"/>
<path class="stroke-thin" d="M24 196h300a22 22 0 0 1 22 22v78a22 22 0 0 1-22 22H120l-52 44 9-44H24a22 22 0 0 1-22-22v-78a22 22 0 0 1 22-22z"/>
<path class="stroke-thin" d="M34 232h250M34 262h210M34 292h150"/>
<g class="op ink-blue"><circle cx="404" cy="254" r="52"/></g>
<path class="stroke" d="M380 254l18 20 32-42"/>
</svg>
</div>
</div>
</section>
<div class="wrap"><div class="rule"></div></div>
<!-- ================= 3. THE LOOP ================= -->
<section class="wrap rise" style="text-align:center">
<span class="tag">The fix</span>
<h2 style="margin:20px auto 14px;max-width:14ch">So we make it practice.</h2>
<p style="margin:0 auto 10px">Four steps, on repeat. That's reinforcement learning.</p>
<svg class="art art-wide" style="margin-top:24px" viewBox="0 0 600 600" role="img" aria-label="A loop of four steps: try, score, nudge, again — repeated millions of times">
<g class="spin">
<g>
<path class="stroke" d="M389.2 132.2A190 190 0 0 1 467.8 210.8"/>
<polygon class="ink-dark" points="0,-11 22,0 0,11" transform="translate(467.8,210.8) rotate(62)"/>
</g>
<g transform="rotate(90 300 300)">
<path class="stroke" d="M389.2 132.2A190 190 0 0 1 467.8 210.8"/>
<polygon class="ink-dark" points="0,-11 22,0 0,11" transform="translate(467.8,210.8) rotate(62)"/>
</g>
<g transform="rotate(180 300 300)">
<path class="stroke" d="M389.2 132.2A190 190 0 0 1 467.8 210.8"/>
<polygon class="ink-dark" points="0,-11 22,0 0,11" transform="translate(467.8,210.8) rotate(62)"/>
</g>
<g transform="rotate(270 300 300)">
<path class="stroke" d="M389.2 132.2A190 190 0 0 1 467.8 210.8"/>
<polygon class="ink-dark" points="0,-11 22,0 0,11" transform="translate(467.8,210.8) rotate(62)"/>
</g>
</g>
<circle class="ink-yellow op" cx="300" cy="110" r="66"/>
<circle class="ink-pink op" cx="490" cy="300" r="66"/>
<circle class="ink-blue op" cx="300" cy="490" r="66"/>
<circle class="ink-yellow op" cx="110" cy="300" r="66"/>
<g class="stroke">
<circle cx="300" cy="110" r="66"/><circle cx="490" cy="300" r="66"/>
<circle cx="300" cy="490" r="66"/><circle cx="110" cy="300" r="66"/>
</g>
<text x="300" y="102" text-anchor="middle" font-size="34">TRY</text>
<text class="mono" x="300" y="132" text-anchor="middle" font-size="15">3 answers</text>
<text x="490" y="292" text-anchor="middle" font-size="34">SCORE</text>
<text class="mono" x="490" y="322" text-anchor="middle" font-size="15">rate them</text>
<text x="300" y="482" text-anchor="middle" font-size="34">NUDGE</text>
<text class="mono" x="300" y="512" text-anchor="middle" font-size="15">toward best</text>
<text x="110" y="292" text-anchor="middle" font-size="34">AGAIN</text>
<text class="mono" x="110" y="322" text-anchor="middle" font-size="15">and again</text>
<circle class="ink-pink op" cx="300" cy="300" r="72"/>
<circle class="stroke" cx="300" cy="300" r="72"/>
<text class="mono" x="300" y="294" text-anchor="middle" font-size="17">millions</text>
<text class="mono" x="300" y="318" text-anchor="middle" font-size="17">of times</text>
</svg>
</section>
<div class="wrap"><div class="rule"></div></div>
<!-- ================= 4. WHO SCORES ================= -->
<section class="wrap rise">
<span class="tag">The judge</span>
<h2 style="margin-top:20px">Who hands out the score?</h2>
<p>Two kinds of judge, depending on the question.</p>
<div class="duo">
<div class="card">
<svg class="art" viewBox="0 0 320 190" role="img" aria-label="A thumbs up and a thumbs down">
<g class="op ink-blue"><circle cx="88" cy="95" r="70"/></g>
<path class="stroke" d="M62 108V80h18l16-30a14 14 0 0 1 20 14l-6 16h20a10 10 0 0 1 10 12l-8 30a12 12 0 0 1-12 9H80a18 18 0 0 1-18-18z"/>
<g class="op ink-pink"><circle cx="232" cy="95" r="70"/></g>
<path class="stroke" d="M206 82v28h18l16 30a14 14 0 0 0 20-14l-6-16h20a10 10 0 0 0 10-12l-8-30a12 12 0 0 0-12-9h-40a18 18 0 0 0-18 18z"/>
</svg>
<h3>A person</h3>
<p>People pick which answer they liked. A second model copies their taste, then does the rating at scale.</p>
</div>
<div class="card">
<svg class="art" viewBox="0 0 320 190" role="img" aria-label="A sum and a code test, each marked correct or not">
<g class="op ink-yellow"><rect x="20" y="30" width="130" height="130" rx="18"/></g>
<rect class="stroke" x="20" y="30" width="130" height="130" rx="18"/>
<text x="85" y="112" text-anchor="middle" font-size="46">2+2</text>
<g class="op ink-blue"><rect x="176" y="30" width="130" height="130" rx="18"/></g>
<rect class="stroke" x="176" y="30" width="130" height="130" rx="18"/>
<path class="stroke" d="M214 78l-16 17 16 17M268 78l16 17-16 17M248 70l-14 50"/>
</svg>
<h3>A checker</h3>
<p>For maths and code there's no opinion needed. Run it. Right or wrong, that's the score.</p>
</div>
</div>
</section>
<div class="wrap"><div class="rule"></div></div>
<!-- ================= 5. REWARD HACKING ================= -->
<section class="wrap rise">
<div class="split flip">
<div>
<span class="tag">The catch</span>
<h2 style="margin-top:20px">It chases the score, not your meaning.</h2>
<p>If flattery scores well, it learns flattery. If long and confident scores well, it learns long and confident — even when it's wrong.</p>
</div>
<div class="col-art">
<svg class="art" viewBox="0 0 480 300" role="img" aria-label="A score dial pinned at maximum above an empty answer bubble">
<g class="op ink-pink"><path d="M60 210a180 180 0 0 1 360 0z"/></g>
<path class="stroke" d="M60 210a180 180 0 0 1 360 0"/>
<g class="stroke-thin">
<path d="M78 172l26-9M120 108l19-19M186 66l9-25M294 66l-9-25M360 108l-19-19M402 172l-26-9"/>
</g>
<path class="stroke" d="M240 210L392 148"/>
<circle class="ink-dark" cx="240" cy="210" r="16"/>
<text class="mono" x="418" y="132" text-anchor="end" font-size="22">10/10</text>
<g class="op ink-yellow"><path d="M120 236h240a18 18 0 0 1 18 18v18a18 18 0 0 1-18 18H120a18 18 0 0 1-18-18v-18a18 18 0 0 1 18-18z"/></g>
<path class="stroke-thin" d="M120 236h240a18 18 0 0 1 18 18v18a18 18 0 0 1-18 18H120a18 18 0 0 1-18-18v-18a18 18 0 0 1 18-18z"/>
<text class="mono" x="240" y="270" text-anchor="middle" font-size="17">…says nothing</text>
</svg>
</div>
</div>
</section>
<!-- ================= 6. PAYOFF ================= -->
<section class="wrap rise">
<span class="tag">Why it matters</span>
<h2 style="margin-top:20px">Nearly every good habit comes from this loop.</h2>
<div class="duo" style="grid-template-columns:1fr">
<svg class="art art-wide" viewBox="0 0 720 200" role="img" aria-label="Three trained habits: answering the question, saying no, and working step by step">
<g class="op ink-yellow"><circle cx="110" cy="80" r="58"/></g>
<path class="stroke" d="M84 62h52M84 82h52M84 102h30"/>
<text class="mono" x="110" y="178" text-anchor="middle" font-size="17">answers you</text>
<g class="op ink-pink"><circle cx="360" cy="80" r="58"/></g>
<path class="stroke" d="M334 80h52"/>
<text class="mono" x="360" y="178" text-anchor="middle" font-size="17">says no</text>
<g class="op ink-blue"><circle cx="610" cy="80" r="58"/></g>
<path class="stroke" d="M578 106l20-24 20 20 24-38"/>
<text class="mono" x="610" y="178" text-anchor="middle" font-size="17">thinks it through</text>
</svg>
</div>
<p style="margin-top:8px">Reading the internet gave it knowledge. Practice gave it manners.</p>
</section>
<div class="wrap"><div class="rule"></div></div>
<!-- ================= CLOSER ================= -->
<section class="wrap closer rise">
<h2>Try. Score.<br>Nudge. Again.</h2>
<p class="lede" style="margin:22px auto 0">That's the whole idea. Everything else is engineering.</p>
</section>
<script>
const io = new IntersectionObserver((es)=>{
es.forEach(e=>{ if(e.isIntersecting){ e.target.classList.add('in'); io.unobserve(e.target); } });
},{threshold:.12});
document.querySelectorAll('.rise').forEach(el=>io.observe(el));
</script>
</body>
</html>