|
| 1 | +// AI similarity de-dup for blog posts. |
| 2 | +// |
| 3 | +// We embed the candidate topic (title + angle) with Cloudflare Workers |
| 4 | +// AI's BGE base model, fetch the embeddings of recent published posts, |
| 5 | +// and compare cosine similarity. Anything above SIMILARITY_THRESHOLD is |
| 6 | +// considered a duplicate. |
| 7 | +// |
| 8 | +// The model is tiny (~6k neurons / M tokens) so this adds negligible |
| 9 | +// cost to a generation. We cache embeddings on each blog_posts row so |
| 10 | +// we only embed each post once. |
| 11 | + |
| 12 | +const EMBED_MODEL = '@cf/baai/bge-base-en-v1.5'; |
| 13 | +// 0.80 in practice — BGE embeddings cluster same-niche content tightly, |
| 14 | +// so all "SEO" posts share heavy vocabulary overlap. Empirically: |
| 15 | +// - near-verbatim rewrite of an existing post: ~0.83 |
| 16 | +// - same topic family, different angle: ~0.77 |
| 17 | +// - clearly distinct topic, same niche: ~0.73 |
| 18 | +// 0.80 catches the rewrites while letting through family-related but |
| 19 | +// distinct angles. Bump higher if too many family-relateds get blocked. |
| 20 | +const SIMILARITY_THRESHOLD = 0.80; |
| 21 | +const RECENT_POSTS_TO_CHECK = 50; // 50 most recent published posts |
| 22 | +const MAX_TEXT_FOR_EMBED = 1500; // chars; well within model's 512 tokens |
| 23 | + |
| 24 | +// Embed a string. Returns { vector, dims, model } or throws. |
| 25 | +export async function embed(env, text) { |
| 26 | + const trimmed = String(text || '').slice(0, MAX_TEXT_FOR_EMBED); |
| 27 | + if (!trimmed) throw new Error('embed: empty text'); |
| 28 | + const out = await env.AI.run(EMBED_MODEL, { text: [trimmed] }); |
| 29 | + // The model returns { shape: [1, 768], data: [[...]] } or sometimes |
| 30 | + // { data: [...] }. Normalise. |
| 31 | + const vec = Array.isArray(out?.data?.[0]) ? out.data[0] |
| 32 | + : Array.isArray(out?.data) ? out.data |
| 33 | + : null; |
| 34 | + if (!vec || !vec.length) throw new Error('embed: bad model response'); |
| 35 | + return { vector: vec, dims: vec.length, model: EMBED_MODEL }; |
| 36 | +} |
| 37 | + |
| 38 | +// Standard cosine similarity. Both vectors must be the same length. |
| 39 | +export function cosine(a, b) { |
| 40 | + if (!a || !b || a.length !== b.length) return 0; |
| 41 | + let dot = 0, na = 0, nb = 0; |
| 42 | + for (let i = 0; i < a.length; i++) { |
| 43 | + dot += a[i] * b[i]; |
| 44 | + na += a[i] * a[i]; |
| 45 | + nb += b[i] * b[i]; |
| 46 | + } |
| 47 | + if (!na || !nb) return 0; |
| 48 | + return dot / (Math.sqrt(na) * Math.sqrt(nb)); |
| 49 | +} |
| 50 | + |
| 51 | +// Given a candidate topic (key + angle), check whether anything in the |
| 52 | +// recent published-posts pool is too similar. Returns: |
| 53 | +// { duplicate: bool, similarity, against: { slug, title, score } | null, |
| 54 | +// scored: [{slug,title,score}] } |
| 55 | +// |
| 56 | +// `scored` lists the top 5 most-similar existing posts for debugging. |
| 57 | +export async function checkDuplicate(env, { title, angle }) { |
| 58 | + if (!env?.AI || !env?.DB) { |
| 59 | + // Best-effort: if AI binding missing, never block. |
| 60 | + return { duplicate: false, similarity: 0, against: null, scored: [], skipped: 'no_ai_binding' }; |
| 61 | + } |
| 62 | + const candidate = String(title || '') + ' — ' + String(angle || ''); |
| 63 | + let candidateVec; |
| 64 | + try { candidateVec = (await embed(env, candidate)).vector; } |
| 65 | + catch (e) { return { duplicate: false, similarity: 0, against: null, scored: [], error: String(e?.message || e) }; } |
| 66 | + |
| 67 | + const rows = await env.DB.prepare( |
| 68 | + `SELECT slug, title, meta_description, embedding |
| 69 | + FROM blog_posts |
| 70 | + WHERE status = 'published' AND embedding IS NOT NULL |
| 71 | + ORDER BY published_at DESC LIMIT ?` |
| 72 | + ).bind(RECENT_POSTS_TO_CHECK).all().catch(() => ({ results: [] })); |
| 73 | + |
| 74 | + const scored = []; |
| 75 | + for (const r of (rows.results || [])) { |
| 76 | + let v; |
| 77 | + try { v = JSON.parse(r.embedding); } catch { continue; } |
| 78 | + if (!Array.isArray(v) || v.length !== candidateVec.length) continue; |
| 79 | + const score = cosine(candidateVec, v); |
| 80 | + scored.push({ slug: r.slug, title: r.title, score }); |
| 81 | + } |
| 82 | + scored.sort((a, b) => b.score - a.score); |
| 83 | + const top = scored[0] || null; |
| 84 | + return { |
| 85 | + duplicate: !!top && top.score >= SIMILARITY_THRESHOLD, |
| 86 | + similarity: top?.score || 0, |
| 87 | + against: top, |
| 88 | + scored: scored.slice(0, 5), |
| 89 | + threshold: SIMILARITY_THRESHOLD, |
| 90 | + }; |
| 91 | +} |
| 92 | + |
| 93 | +// Pick a non-duplicate topic. Calls `pickFn()` up to `maxTries` times, |
| 94 | +// each call returning a fresh candidate. Returns: |
| 95 | +// { topic, dup, tries, fallback } |
| 96 | +// - topic: the chosen topic (the cleanest available) |
| 97 | +// - dup: similarity check result for the chosen topic |
| 98 | +// - tries: how many candidates we burned through |
| 99 | +// - fallback: true if all candidates were duplicates and we picked the |
| 100 | +// least-similar one anyway (logged so cron keeps publishing) |
| 101 | +export async function pickNonDuplicate(env, pickFn, { maxTries = 5 } = {}) { |
| 102 | + const burned = []; |
| 103 | + for (let i = 0; i < maxTries; i++) { |
| 104 | + const topic = await pickFn(); |
| 105 | + if (!topic) break; |
| 106 | + const dup = await checkDuplicate(env, { title: topic.angle, angle: topic.angle }); |
| 107 | + if (!dup.duplicate) { |
| 108 | + return { topic, dup, tries: i + 1, fallback: false }; |
| 109 | + } |
| 110 | + burned.push({ topic, dup }); |
| 111 | + } |
| 112 | + // Everything was a duplicate. Pick the candidate that was LEAST |
| 113 | + // similar so we still publish. The cron stays alive; admin can audit |
| 114 | + // the warning in the post metadata. |
| 115 | + if (!burned.length) { |
| 116 | + // pickFn returned nothing (empty pool) — give up |
| 117 | + return { topic: null, dup: null, tries: maxTries, fallback: true }; |
| 118 | + } |
| 119 | + burned.sort((a, b) => a.dup.similarity - b.dup.similarity); |
| 120 | + const best = burned[0]; |
| 121 | + return { topic: best.topic, dup: best.dup, tries: burned.length, fallback: true }; |
| 122 | +} |
| 123 | + |
| 124 | +// Embed + persist a post's embedding. Called from publish.js so every |
| 125 | +// new post immediately participates in future dedup checks. |
| 126 | +export async function storeEmbedding(env, slug, { title, body_markdown, meta_description }) { |
| 127 | + if (!env?.AI || !env?.DB) return { ok: false, reason: 'no_binding' }; |
| 128 | + // We embed title + meta + first ~1k of body. Captures the topic well |
| 129 | + // without exceeding the model's input limit. |
| 130 | + const body = String(body_markdown || '').slice(0, 800); |
| 131 | + const text = `${title || ''}\n${meta_description || ''}\n${body}`; |
| 132 | + try { |
| 133 | + const { vector, model } = await embed(env, text); |
| 134 | + await env.DB.prepare( |
| 135 | + `UPDATE blog_posts SET embedding = ?, embedding_model = ?, embedding_at = ? |
| 136 | + WHERE slug = ?` |
| 137 | + ).bind(JSON.stringify(vector), model, Math.floor(Date.now() / 1000), slug).run(); |
| 138 | + return { ok: true, dims: vector.length }; |
| 139 | + } catch (e) { |
| 140 | + return { ok: false, reason: String(e?.message || e).slice(0, 200) }; |
| 141 | + } |
| 142 | +} |
0 commit comments