|
| 1 | +<!DOCTYPE html> |
| 2 | +<html lang="en"> |
| 3 | +<head> |
| 4 | +<meta charset="utf-8"> |
| 5 | +<meta name="viewport" content="width=device-width, initial-scale=1"> |
| 6 | +<title>The KL penalty — a leash for a language model</title> |
| 7 | +<link rel="preconnect" href="https://fonts.googleapis.com"> |
| 8 | +<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin> |
| 9 | +<link href="https://fonts.googleapis.com/css2?family=Bricolage+Grotesque:opsz,wght@12..96,600;12..96,800&family=Inter:wght@400;500;600&family=Space+Mono:wght@400;700&display=swap" rel="stylesheet"> |
| 10 | +<style> |
| 11 | + :root{ |
| 12 | + --paper:#DDE2EE; |
| 13 | + --card:#F7F8FC; |
| 14 | + --ink:#141B33; |
| 15 | + --muted:#5A6484; |
| 16 | + --old:#2B3A67; |
| 17 | + --new:#6E3FD3; |
| 18 | + --reward:#E9A13B; |
| 19 | + --bad:#CE3350; |
| 20 | + --display:"Bricolage Grotesque","Arial Black",system-ui,sans-serif; |
| 21 | + --body:"Inter",system-ui,-apple-system,sans-serif; |
| 22 | + --mono:"Space Mono",ui-monospace,monospace; |
| 23 | + } |
| 24 | + |
| 25 | + *{box-sizing:border-box} |
| 26 | + |
| 27 | + html{ |
| 28 | + scroll-snap-type:y proximity; |
| 29 | + scroll-behavior:smooth; |
| 30 | + } |
| 31 | + |
| 32 | + body{ |
| 33 | + margin:0; |
| 34 | + background-color:var(--paper); |
| 35 | + background-image:radial-gradient(circle, rgba(20,27,51,.10) 1px, transparent 1px); |
| 36 | + background-size:26px 26px; |
| 37 | + color:var(--ink); |
| 38 | + font-family:var(--body); |
| 39 | + -webkit-font-smoothing:antialiased; |
| 40 | + } |
| 41 | + |
| 42 | + /* ---- the cord: a leash running down the left edge, doubling as progress ---- */ |
| 43 | + .rail{ |
| 44 | + position:fixed; left:20px; top:0; bottom:0; width:14px; |
| 45 | + z-index:20; pointer-events:none; |
| 46 | + } |
| 47 | + .rail-line{ |
| 48 | + position:absolute; left:6px; top:0; bottom:0; width:2px; |
| 49 | + background:repeating-linear-gradient(to bottom, var(--ink) 0 6px, transparent 6px 12px); |
| 50 | + opacity:.30; |
| 51 | + } |
| 52 | + .rail-knot{ |
| 53 | + position:absolute; left:0; width:14px; height:14px; border-radius:50%; |
| 54 | + background:var(--new); border:2px solid var(--paper); |
| 55 | + top:0; transform:translateY(-50%); |
| 56 | + } |
| 57 | + |
| 58 | + /* ---- panels ---- */ |
| 59 | + .panel{ |
| 60 | + min-height:100dvh; |
| 61 | + scroll-snap-align:center; |
| 62 | + display:flex; flex-direction:column; align-items:center; justify-content:center; |
| 63 | + gap:30px; |
| 64 | + padding:70px 26px 70px 56px; |
| 65 | + text-align:center; |
| 66 | + } |
| 67 | + |
| 68 | + .eyebrow{ |
| 69 | + font-family:var(--mono); font-size:.66rem; letter-spacing:.2em; |
| 70 | + text-transform:uppercase; color:var(--muted); margin:0; |
| 71 | + } |
| 72 | + |
| 73 | + h1{ |
| 74 | + font-family:var(--display); font-weight:800; |
| 75 | + font-size:clamp(3.2rem,15vw,9rem); line-height:.84; letter-spacing:-.045em; |
| 76 | + margin:0; |
| 77 | + } |
| 78 | + h2{ |
| 79 | + font-family:var(--display); font-weight:800; |
| 80 | + font-size:clamp(1.9rem,6.4vw,3.5rem); line-height:1.0; letter-spacing:-.03em; |
| 81 | + margin:0; max-width:16ch; |
| 82 | + } |
| 83 | + .note{ |
| 84 | + font-size:1rem; line-height:1.5; color:var(--muted); |
| 85 | + max-width:34ch; margin:0; |
| 86 | + } |
| 87 | + .stage{width:100%; max-width:720px} |
| 88 | + .stage svg{width:100%; height:auto; display:block; overflow:visible} |
| 89 | + .txt{font-family:var(--body); font-size:21px; font-weight:500; fill:var(--ink)} |
| 90 | + .txt-sm{font-family:var(--mono); font-size:15px; letter-spacing:.08em; fill:var(--muted)} |
| 91 | + |
| 92 | + /* ---- probability bars ---- */ |
| 93 | + .charts{display:flex; flex-wrap:wrap; gap:18px; justify-content:center; width:100%; max-width:720px} |
| 94 | + .chart{ |
| 95 | + flex:1 1 280px; background:var(--card); border:2px solid var(--ink); |
| 96 | + border-radius:20px; padding:20px 18px 14px; |
| 97 | + } |
| 98 | + .chart-head{display:flex; justify-content:space-between; align-items:baseline; margin-bottom:16px} |
| 99 | + .chart-head b{font-family:var(--mono); font-size:.8rem; letter-spacing:.06em} |
| 100 | + .kl{font-family:var(--mono); font-size:.8rem; padding:3px 9px; border-radius:99px; color:#fff} |
| 101 | + .kl.low{background:var(--old)} |
| 102 | + .kl.high{background:var(--bad)} |
| 103 | + .bars{display:flex; align-items:flex-end; justify-content:space-around; height:120px; gap:6px} |
| 104 | + .grp{display:flex; align-items:flex-end; gap:3px} |
| 105 | + .bar{width:13px; border-radius:3px 3px 0 0; min-height:3px} |
| 106 | + .bar.b4{background:var(--old)} |
| 107 | + .bar.aft{background:var(--new)} |
| 108 | + .xlab{display:flex; justify-content:space-around; gap:6px; margin-top:8px} |
| 109 | + .xlab span{font-family:var(--mono); font-size:.62rem; color:var(--muted); flex:1} |
| 110 | + .legend{display:flex; gap:18px; font-family:var(--mono); font-size:.68rem; color:var(--muted); letter-spacing:.08em} |
| 111 | + .legend i{display:inline-block; width:11px; height:11px; border-radius:3px; margin-right:6px; vertical-align:-1px} |
| 112 | + |
| 113 | + /* ---- the knob ---- */ |
| 114 | + .knob{ |
| 115 | + width:100%; max-width:680px; background:var(--card); |
| 116 | + border:2px solid var(--ink); border-radius:24px; padding:26px 22px 22px; |
| 117 | + text-align:left; |
| 118 | + } |
| 119 | + .track{position:relative; height:96px; margin:6px 0 10px} |
| 120 | + .anchor{ |
| 121 | + position:absolute; left:0; top:26px; width:34px; height:34px; border-radius:50%; |
| 122 | + background:var(--old); |
| 123 | + } |
| 124 | + .goal{ |
| 125 | + position:absolute; right:0; top:26px; width:34px; height:34px; border-radius:9px; |
| 126 | + background:var(--reward); transform:rotate(45deg); |
| 127 | + } |
| 128 | + .leash{ |
| 129 | + position:absolute; left:34px; top:42px; height:3px; background:var(--ink); opacity:.5; |
| 130 | + } |
| 131 | + .runner{ |
| 132 | + position:absolute; top:22px; width:42px; height:42px; border-radius:50%; |
| 133 | + background:var(--new); border:3px solid var(--card); |
| 134 | + transform:translateX(-50%); |
| 135 | + } |
| 136 | + .endcap{position:absolute; bottom:0; font-family:var(--mono); font-size:.6rem; letter-spacing:.1em; color:var(--muted)} |
| 137 | + .endcap.l{left:0} .endcap.r{right:0} |
| 138 | + |
| 139 | + .knob input[type=range]{width:100%; accent-color:var(--new); height:26px; cursor:grab} |
| 140 | + .knob input[type=range]:focus-visible{outline:3px solid var(--new); outline-offset:4px; border-radius:6px} |
| 141 | + |
| 142 | + .readout{display:flex; justify-content:space-between; align-items:center; gap:10px; flex-wrap:wrap; margin-bottom:14px} |
| 143 | + .readout b{font-family:var(--mono); font-size:.78rem; letter-spacing:.06em; color:var(--muted)} |
| 144 | + .beta{font-family:var(--display); font-weight:800; font-size:1.6rem; letter-spacing:-.02em} |
| 145 | + |
| 146 | + .verdict{ |
| 147 | + border-radius:16px; padding:16px 18px; border:2px solid var(--ink); |
| 148 | + background:var(--paper); |
| 149 | + } |
| 150 | + .verdict .tag{font-family:var(--mono); font-size:.64rem; letter-spacing:.14em; text-transform:uppercase; display:block; margin-bottom:8px} |
| 151 | + .verdict .say{font-size:1.05rem; line-height:1.45; font-weight:500; margin:0} |
| 152 | + |
| 153 | + .final{font-family:var(--display); font-weight:800; font-size:clamp(2rem,7vw,4.2rem); line-height:.98; letter-spacing:-.035em; max-width:14ch; margin:0} |
| 154 | + |
| 155 | + @media (prefers-reduced-motion: no-preference){ |
| 156 | + .runner,.leash{transition:left .18s ease, width .18s ease} |
| 157 | + } |
| 158 | + @media (max-width:520px){ |
| 159 | + .rail{left:10px} |
| 160 | + .panel{padding-left:40px} |
| 161 | + } |
| 162 | +</style> |
| 163 | +</head> |
| 164 | +<body> |
| 165 | + |
| 166 | +<div class="rail" aria-hidden="true"> |
| 167 | + <div class="rail-line"></div> |
| 168 | + <div class="rail-knot" id="knot"></div> |
| 169 | +</div> |
| 170 | + |
| 171 | +<!-- 1 · HERO --> |
| 172 | +<section class="panel"> |
| 173 | + <p class="eyebrow">how AI training keeps itself honest</p> |
| 174 | + <h1>KL<br>penalty</h1> |
| 175 | + <div class="stage" style="max-width:560px"> |
| 176 | + <svg viewBox="0 0 560 190" role="img" aria-label="Two circles joined by a slack cord"> |
| 177 | + <path d="M160 90 Q280 175 400 90" fill="none" stroke="var(--ink)" stroke-width="6" stroke-linecap="round" opacity=".55"/> |
| 178 | + <circle cx="140" cy="90" r="46" fill="var(--old)"/> |
| 179 | + <circle cx="420" cy="90" r="46" fill="var(--new)"/> |
| 180 | + </svg> |
| 181 | + </div> |
| 182 | + <p class="note">A leash between the model you had and the model you're making.</p> |
| 183 | +</section> |
| 184 | + |
| 185 | +<!-- 2 · THE MODEL YOU START WITH --> |
| 186 | +<section class="panel"> |
| 187 | + <p class="eyebrow">step one</p> |
| 188 | + <h2>You start with a model that already works.</h2> |
| 189 | + <div class="stage"> |
| 190 | + <svg viewBox="0 0 620 250" role="img" aria-label="A model answering a question sensibly"> |
| 191 | + <circle cx="120" cy="125" r="58" fill="var(--old)"/> |
| 192 | + <polygon points="254,110 214,132 254,156" fill="var(--card)" stroke="var(--old)" stroke-width="3"/> |
| 193 | + <rect x="250" y="70" width="350" height="112" rx="26" fill="var(--card)" stroke="var(--old)" stroke-width="3"/> |
| 194 | + <text class="txt-sm" x="276" y="104">how long do i boil an egg?</text> |
| 195 | + <text class="txt" x="276" y="146">Boil it for about 7 minutes.</text> |
| 196 | + </svg> |
| 197 | + </div> |
| 198 | +</section> |
| 199 | + |
| 200 | +<!-- 3 · TRAINING PUSHES IT --> |
| 201 | +<section class="panel"> |
| 202 | + <p class="eyebrow">step two</p> |
| 203 | + <h2>Then you train it to score higher.</h2> |
| 204 | + <div class="stage"> |
| 205 | + <svg viewBox="0 0 620 250" role="img" aria-label="The model being pushed toward a reward star"> |
| 206 | + <circle cx="110" cy="125" r="52" fill="var(--new)"/> |
| 207 | + <path d="M185 125 H430" stroke="var(--ink)" stroke-width="6" stroke-dasharray="14 12" stroke-linecap="round"/> |
| 208 | + <polygon points="455,125 425,108 425,142" fill="var(--ink)"/> |
| 209 | + <polygon points="530,62 552,112 606,118 566,155 577,208 530,181 483,208 494,155 454,118 508,112" |
| 210 | + fill="var(--reward)"/> |
| 211 | + <text class="txt-sm" x="530" y="240" text-anchor="middle">higher score</text> |
| 212 | + </svg> |
| 213 | + </div> |
| 214 | + <p class="note">Good answers get points. It learns to chase them.</p> |
| 215 | +</section> |
| 216 | + |
| 217 | +<!-- 4 · IT BREAKS --> |
| 218 | +<section class="panel"> |
| 219 | + <p class="eyebrow">the problem</p> |
| 220 | + <h2>Chase points too hard, and it stops making sense.</h2> |
| 221 | + <div class="stage"> |
| 222 | + <svg viewBox="0 0 620 250" role="img" aria-label="The model drifted far away, producing nonsense"> |
| 223 | + <circle cx="80" cy="125" r="40" fill="none" stroke="var(--old)" stroke-width="4" stroke-dasharray="8 8" opacity=".5"/> |
| 224 | + <path d="M130 125 H210" stroke="var(--bad)" stroke-width="5" stroke-dasharray="12 10" stroke-linecap="round"/> |
| 225 | + <circle cx="262" cy="125" r="46" fill="var(--bad)"/> |
| 226 | + <polygon points="368,110 328,132 368,156" fill="var(--card)" stroke="var(--bad)" stroke-width="3"/> |
| 227 | + <rect x="364" y="70" width="248" height="112" rx="26" fill="var(--card)" stroke="var(--bad)" stroke-width="3"/> |
| 228 | + <text class="txt" x="386" y="118">EGGS!!! BEST</text> |
| 229 | + <text class="txt" x="386" y="150">ANSWER ★★★★★</text> |
| 230 | + <text class="txt-sm" x="80" y="196" text-anchor="middle">was here</text> |
| 231 | + </svg> |
| 232 | + </div> |
| 233 | + <p class="note">It found a cheap trick that scores well. It forgot how to talk.</p> |
| 234 | +</section> |
| 235 | + |
| 236 | +<!-- 5 · THE LEASH --> |
| 237 | +<section class="panel"> |
| 238 | + <p class="eyebrow">the fix</p> |
| 239 | + <h2>So you tie it to its old self.</h2> |
| 240 | + <div class="stage"> |
| 241 | + <svg viewBox="0 0 620 230" role="img" aria-label="A spring connecting the old model to the new one"> |
| 242 | + <circle cx="80" cy="115" r="44" fill="none" stroke="var(--old)" stroke-width="5" stroke-dasharray="9 9"/> |
| 243 | + <path d="M124 115 L155 78 L185 152 L215 78 L245 152 L275 78 L305 152 L335 78 L365 152 L396 115" |
| 244 | + fill="none" stroke="var(--ink)" stroke-width="6" stroke-linejoin="round" stroke-linecap="round"/> |
| 245 | + <circle cx="440" cy="115" r="44" fill="var(--new)"/> |
| 246 | + <text class="txt-sm" x="260" y="205" text-anchor="middle">pulls back</text> |
| 247 | + </svg> |
| 248 | + </div> |
| 249 | + <p class="note">It can move. It just has to pull against something.</p> |
| 250 | +</section> |
| 251 | + |
| 252 | +<!-- 6 · WHAT KL MEASURES --> |
| 253 | +<section class="panel"> |
| 254 | + <p class="eyebrow">what "KL" actually is</p> |
| 255 | + <h2>A number for how much it changed.</h2> |
| 256 | + <div class="legend"> |
| 257 | + <span><i style="background:var(--old)"></i>before</span> |
| 258 | + <span><i style="background:var(--new)"></i>after</span> |
| 259 | + </div> |
| 260 | + <div class="charts"> |
| 261 | + <div class="chart"> |
| 262 | + <div class="chart-head"><b>moved a little</b><span class="kl low">KL 0.03</span></div> |
| 263 | + <div class="bars"> |
| 264 | + <div class="grp"><div class="bar b4" style="height:92%"></div><div class="bar aft" style="height:88%"></div></div> |
| 265 | + <div class="grp"><div class="bar b4" style="height:5%"></div><div class="bar aft" style="height:7%"></div></div> |
| 266 | + <div class="grp"><div class="bar b4" style="height:3%"></div><div class="bar aft" style="height:4%"></div></div> |
| 267 | + <div class="grp"><div class="bar b4" style="height:2%"></div><div class="bar aft" style="height:3%"></div></div> |
| 268 | + </div> |
| 269 | + <div class="xlab"><span>Paris</span><span>the</span><span>a</span><span>in</span></div> |
| 270 | + </div> |
| 271 | + <div class="chart"> |
| 272 | + <div class="chart-head"><b>moved a lot</b><span class="kl high">KL 2.41</span></div> |
| 273 | + <div class="bars"> |
| 274 | + <div class="grp"><div class="bar b4" style="height:92%"></div><div class="bar aft" style="height:9%"></div></div> |
| 275 | + <div class="grp"><div class="bar b4" style="height:5%"></div><div class="bar aft" style="height:14%"></div></div> |
| 276 | + <div class="grp"><div class="bar b4" style="height:3%"></div><div class="bar aft" style="height:6%"></div></div> |
| 277 | + <div class="grp"><div class="bar b4" style="height:2%"></div><div class="bar aft" style="height:78%"></div></div> |
| 278 | + </div> |
| 279 | + <div class="xlab"><span>Paris</span><span>the</span><span>a</span><span>in</span></div> |
| 280 | + </div> |
| 281 | + </div> |
| 282 | + <p class="note">Same question. Which word does it reach for now? Big gap, big number.</p> |
| 283 | +</section> |
| 284 | + |
| 285 | +<!-- 7 · THE PRICE --> |
| 286 | +<section class="panel"> |
| 287 | + <p class="eyebrow">the deal</p> |
| 288 | + <div class="stage" style="max-width:640px"> |
| 289 | + <svg viewBox="0 0 640 170" role="img" aria-label="Score minus beta times distance"> |
| 290 | + <text x="0" y="86" font-family="var(--display)" font-weight="800" font-size="52" fill="var(--reward)">score</text> |
| 291 | + <text x="182" y="86" font-family="var(--display)" font-weight="800" font-size="52" fill="var(--ink)">−</text> |
| 292 | + <text x="238" y="86" font-family="var(--display)" font-weight="800" font-size="52" fill="var(--new)">β</text> |
| 293 | + <text x="288" y="86" font-family="var(--display)" font-weight="800" font-size="52" fill="var(--ink)">×</text> |
| 294 | + <text x="348" y="86" font-family="var(--display)" font-weight="800" font-size="52" fill="var(--old)">how far it moved</text> |
| 295 | + <text class="txt-sm" x="0" y="130">points it earned</text> |
| 296 | + <text class="txt-sm" x="348" y="130">points it pays</text> |
| 297 | + </svg> |
| 298 | + </div> |
| 299 | + <h2>It only drifts when the drift is worth paying for.</h2> |
| 300 | +</section> |
| 301 | + |
| 302 | +<!-- 8 · THE KNOB --> |
| 303 | +<section class="panel"> |
| 304 | + <p class="eyebrow">try it</p> |
| 305 | + <h2>β sets how short the leash is.</h2> |
| 306 | + <div class="knob"> |
| 307 | + <div class="track"> |
| 308 | + <div class="anchor" title="old self"></div> |
| 309 | + <div class="leash" id="leash"></div> |
| 310 | + <div class="runner" id="runner"></div> |
| 311 | + <div class="goal"></div> |
| 312 | + <span class="endcap l">OLD SELF</span> |
| 313 | + <span class="endcap r">MAX SCORE</span> |
| 314 | + </div> |
| 315 | + <label for="beta" style="display:block"> |
| 316 | + <input type="range" id="beta" min="0" max="10" step="0.1" value="1.6" aria-label="Strength of the KL penalty, beta"> |
| 317 | + </label> |
| 318 | + <div class="readout"> |
| 319 | + <span><b>β</b> <span class="beta" id="betaOut">1.6</span></span> |
| 320 | + <span><b>distance from old self</b> <span class="beta" id="klOut">2.10</span></span> |
| 321 | + </div> |
| 322 | + <div class="verdict"> |
| 323 | + <span class="tag" id="tag">balanced</span> |
| 324 | + <p class="say" id="say">About 7 minutes for a firm yolk — 6 if you like it jammy.</p> |
| 325 | + </div> |
| 326 | + </div> |
| 327 | + <p class="note">Drag it. Loose leash, nonsense. Tight leash, nothing learned.</p> |
| 328 | +</section> |
| 329 | + |
| 330 | +<!-- 9 · CLOSE --> |
| 331 | +<section class="panel"> |
| 332 | + <div class="stage" style="max-width:340px"> |
| 333 | + <svg viewBox="0 0 340 120" role="img" aria-label="Two circles joined by a short cord"> |
| 334 | + <path d="M110 60 Q170 92 230 60" fill="none" stroke="var(--ink)" stroke-width="5" stroke-linecap="round" opacity=".55"/> |
| 335 | + <circle cx="90" cy="60" r="30" fill="var(--old)"/> |
| 336 | + <circle cx="250" cy="60" r="30" fill="var(--new)"/> |
| 337 | + </svg> |
| 338 | + </div> |
| 339 | + <p class="final">Improve. Don't become a stranger.</p> |
| 340 | + <p class="eyebrow">that's the whole idea</p> |
| 341 | +</section> |
| 342 | + |
| 343 | +<script> |
| 344 | +(function(){ |
| 345 | + var beta = document.getElementById('beta'); |
| 346 | + var runner = document.getElementById('runner'); |
| 347 | + var leash = document.getElementById('leash'); |
| 348 | + var bOut = document.getElementById('betaOut'); |
| 349 | + var kOut = document.getElementById('klOut'); |
| 350 | + var tag = document.getElementById('tag'); |
| 351 | + var say = document.getElementById('say'); |
| 352 | + |
| 353 | + var STATES = { |
| 354 | + loose: { tag:'leash too loose · gaming the score', color:'var(--bad)', |
| 355 | + text:'EGGS!!! BEST ANSWER ★★★★★ EGGS!!! SO HELPFUL!!!' }, |
| 356 | + good: { tag:'balanced · better, still itself', color:'var(--new)', |
| 357 | + text:'About 7 minutes for a firm yolk — 6 if you like it jammy.' }, |
| 358 | + tight: { tag:'leash too tight · nothing learned', color:'var(--old)', |
| 359 | + text:'Boil it for about 7 minutes.' } |
| 360 | + }; |
| 361 | + |
| 362 | + function update(){ |
| 363 | + var b = parseFloat(beta.value); |
| 364 | + var pos = Math.max(7, 100 / (1 + 0.45 * b)); // % across the track |
| 365 | + var kl = Math.min(12, 6 / (0.28 + b)); |
| 366 | + |
| 367 | + runner.style.left = pos + '%'; |
| 368 | + leash.style.width = 'calc(' + pos + '% - 34px)'; |
| 369 | + bOut.textContent = b.toFixed(1); |
| 370 | + kOut.textContent = kl.toFixed(2); |
| 371 | + |
| 372 | + var s = b < 0.5 ? STATES.loose : (b > 4 ? STATES.tight : STATES.good); |
| 373 | + runner.style.background = s.color; |
| 374 | + tag.textContent = s.tag; |
| 375 | + tag.style.color = s.color; |
| 376 | + say.textContent = s.text; |
| 377 | + } |
| 378 | + |
| 379 | + beta.addEventListener('input', update); |
| 380 | + update(); |
| 381 | + |
| 382 | + // leash knot tracks scroll progress |
| 383 | + var knot = document.getElementById('knot'); |
| 384 | + function track(){ |
| 385 | + var max = document.documentElement.scrollHeight - window.innerHeight; |
| 386 | + var p = max > 0 ? window.scrollY / max : 0; |
| 387 | + knot.style.top = (p * (window.innerHeight - 20) + 10) + 'px'; |
| 388 | + } |
| 389 | + window.addEventListener('scroll', track, {passive:true}); |
| 390 | + window.addEventListener('resize', track); |
| 391 | + track(); |
| 392 | +})(); |
| 393 | +</script> |
| 394 | +</body> |
| 395 | +</html> |
0 commit comments