|
| 1 | +//! Backend-agnostic token sampling. |
| 2 | +//! |
| 3 | +//! Implements the standard HF / llama.cpp-style sampling pipeline: |
| 4 | +//! |
| 5 | +//! 1. Repeat penalty over a sliding window of recently emitted tokens |
| 6 | +//! (divide positive logits, multiply negative — pushes mass off |
| 7 | +//! self-reinforcing attractors that cause infinite repetition on |
| 8 | +//! aggressively quantized models). |
| 9 | +//! 2. Temperature scaling (`logits /= T`). |
| 10 | +//! 3. Softmax with the max-subtraction stability trick. |
| 11 | +//! 4. Top-p nucleus filtering. |
| 12 | +//! 5. Multinomial draw via a deterministic xorshift64* PRNG. |
| 13 | +//! |
| 14 | +//! The whole pipeline operates on `&mut [f32]` so it can be shared |
| 15 | +//! across backends: the GPU runtime (mlx, candle) is responsible for |
| 16 | +//! computing per-step logits and pulling the last-position vector into |
| 17 | +//! a CPU buffer; from there `sample_from_logits` returns the next |
| 18 | +//! token id. |
| 19 | +
|
| 20 | +/// Tuning knobs for one sampling step. `is_greedy()` returns true when |
| 21 | +/// the config collapses to argmax (temperature 0 AND no repeat |
| 22 | +/// penalty), letting callers skip the CPU pipeline entirely. |
| 23 | +#[derive(Debug, Clone, Copy)] |
| 24 | +pub struct SamplingConfig { |
| 25 | + /// Softmax temperature. `<= 0` collapses to argmax (greedy). |
| 26 | + pub temperature: f32, |
| 27 | + /// Nucleus sampling cutoff in `(0, 1]`. `>= 1.0` disables top-p |
| 28 | + /// (full distribution). |
| 29 | + pub top_p: f32, |
| 30 | + /// HF / llama.cpp-style repeat penalty applied to the last |
| 31 | + /// `repeat_penalty_last_n` tokens. `1.0` = no penalty. |
| 32 | + pub repeat_penalty: f32, |
| 33 | + /// Sliding-window length the penalty applies to. `0` disables. |
| 34 | + pub repeat_penalty_last_n: usize, |
| 35 | + /// PRNG seed. Same prompt + seed → bit-identical output. |
| 36 | + pub seed: u64, |
| 37 | +} |
| 38 | + |
| 39 | +impl Default for SamplingConfig { |
| 40 | + fn default() -> Self { |
| 41 | + Self { |
| 42 | + temperature: 0.0, |
| 43 | + top_p: 1.0, |
| 44 | + repeat_penalty: 1.0, |
| 45 | + repeat_penalty_last_n: 64, |
| 46 | + seed: 0, |
| 47 | + } |
| 48 | + } |
| 49 | +} |
| 50 | + |
| 51 | +impl SamplingConfig { |
| 52 | + pub fn is_greedy(&self) -> bool { |
| 53 | + self.temperature <= 0.0 && (self.repeat_penalty - 1.0).abs() < 1e-6 |
| 54 | + } |
| 55 | +} |
| 56 | + |
| 57 | +/// xorshift64* — deterministic, no external crate, perfectly adequate |
| 58 | +/// for token sampling. Cryptographic strength is not a goal; cheap |
| 59 | +/// seeded reproducibility is. |
| 60 | +pub struct Xorshift64 { |
| 61 | + state: u64, |
| 62 | +} |
| 63 | + |
| 64 | +impl Xorshift64 { |
| 65 | + pub fn new(seed: u64) -> Self { |
| 66 | + // Reject the all-zero state — xorshift would lock at 0. |
| 67 | + let state = if seed == 0 { 0x9E3779B97F4A7C15 } else { seed }; |
| 68 | + Self { state } |
| 69 | + } |
| 70 | + |
| 71 | + pub fn next_u64(&mut self) -> u64 { |
| 72 | + let mut x = self.state; |
| 73 | + x ^= x >> 12; |
| 74 | + x ^= x << 25; |
| 75 | + x ^= x >> 27; |
| 76 | + self.state = x; |
| 77 | + x.wrapping_mul(0x2545F4914F6CDD1D) |
| 78 | + } |
| 79 | + |
| 80 | + /// Uniform `f32` in `[0, 1)`. |
| 81 | + pub fn next_f32(&mut self) -> f32 { |
| 82 | + ((self.next_u64() >> 40) as f32) / (1u32 << 24) as f32 |
| 83 | + } |
| 84 | +} |
| 85 | + |
| 86 | +/// HF-style repeat penalty applied in place: divide positive logits and |
| 87 | +/// multiply negative logits of recently-emitted tokens by `penalty`. |
| 88 | +/// Pushes probability mass off repeated-token attractors that |
| 89 | +/// catastrophically dominate greedy decoding on aggressive 3-bit |
| 90 | +/// quantization (the original `karın-karın-...` bug class). |
| 91 | +pub fn apply_repeat_penalty(logits: &mut [f32], recent: &[u32], penalty: f32) { |
| 92 | + if (penalty - 1.0).abs() < 1e-6 { |
| 93 | + return; |
| 94 | + } |
| 95 | + for &tok in recent { |
| 96 | + let i = tok as usize; |
| 97 | + if i >= logits.len() { |
| 98 | + continue; |
| 99 | + } |
| 100 | + let v = logits[i]; |
| 101 | + logits[i] = if v >= 0.0 { v / penalty } else { v * penalty }; |
| 102 | + } |
| 103 | +} |
| 104 | + |
| 105 | +/// In-place softmax with the standard max-subtraction trick for |
| 106 | +/// numerical stability. After this call `logits` is a valid probability |
| 107 | +/// distribution summing to ~1.0. Falls back to uniform on degenerate |
| 108 | +/// input (all `-inf`) instead of producing NaNs. |
| 109 | +pub fn softmax_inplace(logits: &mut [f32]) { |
| 110 | + let max = logits.iter().copied().fold(f32::NEG_INFINITY, f32::max); |
| 111 | + if !max.is_finite() { |
| 112 | + let u = 1.0 / logits.len() as f32; |
| 113 | + for v in logits.iter_mut() { |
| 114 | + *v = u; |
| 115 | + } |
| 116 | + return; |
| 117 | + } |
| 118 | + let mut sum = 0.0_f32; |
| 119 | + for v in logits.iter_mut() { |
| 120 | + *v = (*v - max).exp(); |
| 121 | + sum += *v; |
| 122 | + } |
| 123 | + if sum <= 0.0 { |
| 124 | + let u = 1.0 / logits.len() as f32; |
| 125 | + for v in logits.iter_mut() { |
| 126 | + *v = u; |
| 127 | + } |
| 128 | + return; |
| 129 | + } |
| 130 | + let inv = 1.0 / sum; |
| 131 | + for v in logits.iter_mut() { |
| 132 | + *v *= inv; |
| 133 | + } |
| 134 | +} |
| 135 | + |
| 136 | +/// Sample a token id from `probs` after applying top-p nucleus |
| 137 | +/// filtering. `probs` must sum to ~1 (call `softmax_inplace` first). |
| 138 | +/// `top_p >= 1.0` skips the filter and samples from the full |
| 139 | +/// distribution. Always keeps at least one token (the argmax) so a |
| 140 | +/// degenerate `top_p = 0` doesn't deadlock. |
| 141 | +pub fn sample_top_p(probs: &[f32], top_p: f32, rng: &mut Xorshift64) -> u32 { |
| 142 | + debug_assert!(!probs.is_empty()); |
| 143 | + |
| 144 | + if top_p >= 1.0 || top_p <= 0.0 { |
| 145 | + return categorical(probs, rng); |
| 146 | + } |
| 147 | + |
| 148 | + let mut idx: Vec<u32> = (0..probs.len() as u32).collect(); |
| 149 | + idx.sort_unstable_by(|&a, &b| { |
| 150 | + probs[b as usize] |
| 151 | + .partial_cmp(&probs[a as usize]) |
| 152 | + .unwrap_or(std::cmp::Ordering::Equal) |
| 153 | + }); |
| 154 | + |
| 155 | + let mut cum = 0.0_f32; |
| 156 | + let mut cutoff = idx.len(); |
| 157 | + for (rank, &i) in idx.iter().enumerate() { |
| 158 | + cum += probs[i as usize]; |
| 159 | + if cum >= top_p { |
| 160 | + cutoff = rank + 1; |
| 161 | + break; |
| 162 | + } |
| 163 | + } |
| 164 | + cutoff = cutoff.max(1); |
| 165 | + |
| 166 | + let kept = &idx[..cutoff]; |
| 167 | + let mass: f32 = kept.iter().map(|&i| probs[i as usize]).sum(); |
| 168 | + if mass <= 0.0 { |
| 169 | + return kept[0]; |
| 170 | + } |
| 171 | + let r = rng.next_f32() * mass; |
| 172 | + let mut acc = 0.0_f32; |
| 173 | + for &i in kept { |
| 174 | + acc += probs[i as usize]; |
| 175 | + if r < acc { |
| 176 | + return i; |
| 177 | + } |
| 178 | + } |
| 179 | + *kept.last().unwrap() |
| 180 | +} |
| 181 | + |
| 182 | +fn categorical(probs: &[f32], rng: &mut Xorshift64) -> u32 { |
| 183 | + let r = rng.next_f32(); |
| 184 | + let mut acc = 0.0_f32; |
| 185 | + for (i, &p) in probs.iter().enumerate() { |
| 186 | + acc += p; |
| 187 | + if r < acc { |
| 188 | + return i as u32; |
| 189 | + } |
| 190 | + } |
| 191 | + (probs.len() - 1) as u32 |
| 192 | +} |
| 193 | + |
| 194 | +/// One-shot helper: run the full pipeline (penalty → temperature → |
| 195 | +/// softmax → top-p → sample) on a CPU logit buffer. Mutates `logits` |
| 196 | +/// in place (caller may discard or reuse). The caller owns |
| 197 | +/// `recent_tokens` (sliding window for the repeat penalty). |
| 198 | +pub fn sample_from_logits( |
| 199 | + logits: &mut [f32], |
| 200 | + recent_tokens: &[u32], |
| 201 | + cfg: &SamplingConfig, |
| 202 | + rng: &mut Xorshift64, |
| 203 | +) -> u32 { |
| 204 | + // Repeat penalty restricted to the trailing window — older context |
| 205 | + // shouldn't dampen tokens we naturally want to emit again. |
| 206 | + let n = cfg.repeat_penalty_last_n.min(recent_tokens.len()); |
| 207 | + if cfg.repeat_penalty != 1.0 && n > 0 { |
| 208 | + let window = &recent_tokens[recent_tokens.len() - n..]; |
| 209 | + apply_repeat_penalty(logits, window, cfg.repeat_penalty); |
| 210 | + } |
| 211 | + |
| 212 | + // Temperature scaling before softmax. `<=0` would mean greedy but |
| 213 | + // the caller is responsible for routing greedy elsewhere; clamp to |
| 214 | + // a tiny epsilon as a safety net. |
| 215 | + let t = cfg.temperature.max(1e-5); |
| 216 | + if (t - 1.0).abs() > 1e-6 { |
| 217 | + let inv = 1.0 / t; |
| 218 | + for v in logits.iter_mut() { |
| 219 | + *v *= inv; |
| 220 | + } |
| 221 | + } |
| 222 | + |
| 223 | + softmax_inplace(logits); |
| 224 | + sample_top_p(logits, cfg.top_p, rng) |
| 225 | +} |
| 226 | + |
| 227 | +#[cfg(test)] |
| 228 | +mod tests { |
| 229 | + use super::*; |
| 230 | + |
| 231 | + #[test] |
| 232 | + fn xorshift_deterministic() { |
| 233 | + let mut a = Xorshift64::new(42); |
| 234 | + let mut b = Xorshift64::new(42); |
| 235 | + for _ in 0..16 { |
| 236 | + assert_eq!(a.next_u64(), b.next_u64()); |
| 237 | + } |
| 238 | + } |
| 239 | + |
| 240 | + #[test] |
| 241 | + fn softmax_sums_to_one() { |
| 242 | + let mut v = vec![1.0_f32, 2.0, 3.0, 4.0]; |
| 243 | + softmax_inplace(&mut v); |
| 244 | + let s: f32 = v.iter().sum(); |
| 245 | + assert!((s - 1.0).abs() < 1e-5); |
| 246 | + } |
| 247 | + |
| 248 | + #[test] |
| 249 | + fn softmax_handles_neg_inf() { |
| 250 | + let mut v = vec![f32::NEG_INFINITY; 4]; |
| 251 | + softmax_inplace(&mut v); |
| 252 | + assert!((v.iter().sum::<f32>() - 1.0).abs() < 1e-5); |
| 253 | + } |
| 254 | + |
| 255 | + #[test] |
| 256 | + fn repeat_penalty_pushes_positive_down() { |
| 257 | + let mut v = vec![2.0_f32, 0.5, -0.5, -2.0]; |
| 258 | + apply_repeat_penalty(&mut v, &[0, 2], 1.5); |
| 259 | + assert!(v[0] < 2.0, "positive logit should be divided"); |
| 260 | + assert!(v[2] < -0.5, "negative logit should be multiplied"); |
| 261 | + assert_eq!(v[1], 0.5); |
| 262 | + assert_eq!(v[3], -2.0); |
| 263 | + } |
| 264 | + |
| 265 | + #[test] |
| 266 | + fn top_p_keeps_at_least_one() { |
| 267 | + let probs = vec![0.4, 0.3, 0.2, 0.1]; |
| 268 | + let mut rng = Xorshift64::new(1); |
| 269 | + let tok = sample_top_p(&probs, 0.0, &mut rng); |
| 270 | + assert!(tok < probs.len() as u32); |
| 271 | + } |
| 272 | + |
| 273 | + #[test] |
| 274 | + fn greedy_flag() { |
| 275 | + let g = SamplingConfig::default(); |
| 276 | + assert!(g.is_greedy()); |
| 277 | + let s = SamplingConfig { |
| 278 | + temperature: 0.7, |
| 279 | + ..g |
| 280 | + }; |
| 281 | + assert!(!s.is_greedy()); |
| 282 | + } |
| 283 | + |
| 284 | + #[test] |
| 285 | + fn end_to_end_seeded_is_deterministic() { |
| 286 | + let mut probs = vec![1.0_f32, 2.0, 3.0, 4.0, 5.0]; |
| 287 | + let cfg = SamplingConfig { |
| 288 | + temperature: 0.7, |
| 289 | + top_p: 0.9, |
| 290 | + repeat_penalty: 1.0, |
| 291 | + repeat_penalty_last_n: 0, |
| 292 | + seed: 12345, |
| 293 | + }; |
| 294 | + let mut rng1 = Xorshift64::new(cfg.seed); |
| 295 | + let t1 = sample_from_logits(&mut probs.clone(), &[], &cfg, &mut rng1); |
| 296 | + let mut rng2 = Xorshift64::new(cfg.seed); |
| 297 | + let t2 = sample_from_logits(&mut probs, &[], &cfg, &mut rng2); |
| 298 | + assert_eq!(t1, t2); |
| 299 | + } |
| 300 | +} |
0 commit comments