process Accept-Language for users who have no social graph yet

Author: seboslaw

README.md

@@ -20,7 +20,12 @@ A collaborative filter over the network's **likes**:
    the viewer has already liked or seen.
 
 Anonymous viewers and brand-new accounts get a cold-start popularity feed (most-liked recent posts).
-Every parameter is env-overridable — see [.env.example](.env.example).
+For an authenticated but likeless account, that feed is biased toward the languages in the request's
+`Accept-Language` header (forwarded by the AppView) — the only per-viewer taste signal available before
+the account has liked anything. Posts declaring a non-matching language are dropped; posts that declare
+no language always pass, so the feed narrows toward the viewer's languages without starving. Anonymous
+viewers share one cached list, so they stay global (a per-request header there would poison the shared
+list). Every ranking parameter is env-overridable — see [.env.example](.env.example).
 
 ## Architecture
 

src/algos/for-you.ts

@@ -18,7 +18,8 @@ import {
 
 const cfRanker: Ranker = new CollaborativeFilterRanker()
 const graphRanker: Ranker = new GraphRanker()
-const popularityRanker: Ranker = new PopularityRanker()
+// Concrete type (not Ranker): its rank() takes an extra cold-start language arg.
+const popularityRanker = new PopularityRanker()
 
 // Compute-on-request with a per-(feed, viewer) Redis cache of the ranked list.
 // The cached list is an IMMUTABLE snapshot: a seen-aware order is baked in once
@@ -32,6 +33,10 @@ export const handler = async (
   params: QueryParams,
   viewerDid: string | null,
   feed: FeedDef,
+  // Normalized primary language subtags from the viewer's Accept-Language,
+  // in preference order; [] when the header is absent. Used only to bias the
+  // cold-start feed (see computeRanked).
+  viewerLangs: string[],
 ) => {
   const cacheKey = rankedListKey(feed.rkey, viewerDid)
   const offset = parseCursor(params.cursor)
@@ -41,7 +46,7 @@ export const handler = async (
     // On a cache miss, import the viewer's like history so the first request is
     // already personalized (runs at most once per backfill TTL).
     if (viewerDid) await ensureViewerBackfilled(ctx, viewerDid)
-    const scored = await computeRanked(ctx, viewerDid, feed.content)
+    const scored = await computeRanked(ctx, viewerDid, feed.content, viewerLangs)
     // Bake the seen-aware order in now so every page is a plain offset slice.
     ranked = await orderBySeen(ctx, viewerDid, scored)
     await cacheRankedList(
@@ -91,13 +96,20 @@ const computeRanked = async (
   ctx: AppContext,
   viewerDid: string | null,
   content: ContentFilter,
+  viewerLangs: string[],
 ): Promise<string[]> => {
   // Personalized first; fall back to popularity for anonymous / no-history
   // viewers and while the graph is still building.
   const engine = ctx.cfg.rankerEngine === 'graph' ? graphRanker : cfRanker
   const personalized = await engine.rank(ctx, viewerDid, content)
   if (personalized.length > 0) return personalized
-  return popularityRanker.rank(ctx, viewerDid, content)
+  // Bias the cold-start feed by the viewer's Accept-Language — but only for
+  // authenticated viewers, whose ranked list is cached per-DID. Anonymous
+  // viewers share one cache entry (viewerDid = null), so applying a per-request
+  // header there would let one viewer's language poison every other viewer's
+  // shared list; they stay global.
+  const langs = viewerDid ? viewerLangs : []
+  return popularityRanker.rank(ctx, viewerDid, content, langs)
 }
 
 const parseCursor = (cursor?: string): number => {

src/db/migrations.ts

@@ -147,3 +147,18 @@ migrations['004'] = {
     await db.schema.dropTable('interactions').execute()
   },
 }
+
+migrations['005'] = {
+  async up(db: Kysely<unknown>) {
+    // BCP-47 language subtags declared on the post record, normalized to primary
+    // subtags and comma-joined (e.g. 'en,de'); '' when the post declares none.
+    // Powers the cold-start language allowlist (see ranker/finalize.ts).
+    await db.schema
+      .alterTable('post_meta')
+      .addColumn('langs', 'varchar', (col) => col.notNull().defaultTo(''))
+      .execute()
+  },
+  async down(db: Kysely<unknown>) {
+    await db.schema.alterTable('post_meta').dropColumn('langs').execute()
+  },
+}

src/db/schema.ts

@@ -39,6 +39,10 @@ export type PostMeta = {
   // media flags for content-typed feed variants
   is_image: number
   is_video: number
+  // declared post languages: normalized primary BCP-47 subtags, comma-joined
+  // (e.g. 'en,de'); '' when the post declares none. Feeds the cold-start
+  // language allowlist.
+  langs: string
   // when this metadata was last refreshed (ISO 8601)
   hydrated_at: string
 }

src/methods/feed-generation.ts

@@ -31,10 +31,45 @@ export default function (server: Server, ctx: AppContext) {
       viewerDid = null
     }
 
-    const body = await handler(ctx, params, viewerDid, feed)
+    // The AppView forwards the client's content-language preference here; it's
+    // the only per-viewer taste signal available for a brand-new account with
+    // no likes, so it seeds the cold-start feed's language bias.
+    const viewerLangs = parseAcceptLanguage(req.headers['accept-language'])
+
+    const body = await handler(ctx, params, viewerDid, feed, viewerLangs)
     return {
       encoding: 'application/json',
       body,
     }
   })
 }
+
+// Parses an Accept-Language header ('fr-CH, fr;q=0.9, en;q=0.8, *;q=0.5') into
+// deduped primary BCP-47 subtags in descending q-order ('pt-BR' -> 'pt'). The
+// wildcard '*' and malformed entries are dropped. Returns [] when absent/empty,
+// which leaves the cold-start feed global.
+const parseAcceptLanguage = (header?: string): string[] => {
+  if (!header) return []
+  const ranked = header
+    .split(',')
+    .map((part) => {
+      const [tag, ...paramParts] = part.trim().split(';')
+      const primary = tag.split('-')[0].trim().toLowerCase()
+      const qParam = paramParts
+        .map((p) => p.trim())
+        .find((p) => p.startsWith('q='))
+      const q = qParam ? parseFloat(qParam.slice(2)) : 1
+      return { primary, q: isNaN(q) ? 0 : q }
+    })
+    .filter((e) => e.primary && e.primary !== '*' && e.q > 0)
+    .sort((a, b) => b.q - a.q)
+
+  const out: string[] = []
+  const seen = new Set<string>()
+  for (const { primary } of ranked) {
+    if (seen.has(primary)) continue
+    seen.add(primary)
+    out.push(primary)
+  }
+  return out
+}

src/ranker/finalize.ts

@@ -8,6 +8,11 @@ export type FinalizeOptions = {
   applyPopularityPenalty: boolean
   // restrict results to a media type (image/video) for content-typed variants
   content: ContentFilter
+  // cold-start language allowlist (normalized primary BCP-47 subtags). When
+  // non-empty, a post survives only if it declares no language or shares at
+  // least one with the allowlist — undeclared posts always pass, so the feed
+  // biases toward these languages without starving. Empty/undefined = off.
+  languages?: string[]
 }
 
 // Shared back half of every ranker: hydrate candidate metadata, apply
@@ -24,6 +29,8 @@ export const finalize = async (
   const metas = await hydratePostMeta(ctx, [...rawScores.keys()])
   const now = Date.now()
   const freshnessMs = cfg.freshnessHours * 60 * 60 * 1000
+  const langAllow =
+    opts.languages && opts.languages.length > 0 ? new Set(opts.languages) : null
 
   const scored: { uri: string; author: string; score: number }[] = []
   for (const [uri, raw] of rawScores) {
@@ -33,6 +40,13 @@ export const finalize = async (
     if (meta.is_reply && !cfg.includeReplies) continue // top-level posts only
     if (opts.content === 'image' && !meta.is_image) continue
     if (opts.content === 'video' && !meta.is_video) continue
+    // Language allowlist: undeclared posts pass; declared ones must overlap.
+    if (
+      langAllow &&
+      meta.langs.length > 0 &&
+      !meta.langs.some((l) => langAllow.has(l))
+    )
+      continue
 
     const age = now - Date.parse(meta.created_at)
     if (isNaN(age) || age > freshnessMs) continue

src/ranker/hydrate.ts

@@ -52,6 +52,7 @@ export const hydratePostMeta = async (
           is_reply: m.is_reply ? 1 : 0,
           is_image: m.is_image ? 1 : 0,
           is_video: m.is_video ? 1 : 0,
+          langs: m.langs.join(','),
           hydrated_at: new Date().toISOString(),
         })),
       )
@@ -65,6 +66,7 @@ export const hydratePostMeta = async (
           is_reply: eb.ref('excluded.is_reply'),
           is_image: eb.ref('excluded.is_image'),
           is_video: eb.ref('excluded.is_video'),
+          langs: eb.ref('excluded.langs'),
           hydrated_at: eb.ref('excluded.hydrated_at'),
         })),
       )
@@ -116,6 +118,7 @@ const fetchFromAppview = async (
         is_reply: !!record.reply,
         is_image: hasMedia(post, 'images'),
         is_video: hasMedia(post, 'video'),
+        langs: normalizeLangs(record.langs),
       })
     }
   }
@@ -146,6 +149,23 @@ const hasAdultLabel = (post: any): boolean => {
   return labels.some((l) => ADULT_LABELS.has(l?.val))
 }
 
+// Normalizes a post record's `langs` to deduped primary BCP-47 subtags:
+// lowercased, region/script stripped ('pt-BR' -> 'pt'). Ignores malformed
+// entries. Returns [] when the post declares no usable language.
+const normalizeLangs = (raw: unknown): string[] => {
+  if (!Array.isArray(raw)) return []
+  const out: string[] = []
+  const seen = new Set<string>()
+  for (const entry of raw) {
+    if (typeof entry !== 'string') continue
+    const primary = entry.split('-')[0].trim().toLowerCase()
+    if (!primary || seen.has(primary)) continue
+    seen.add(primary)
+    out.push(primary)
+  }
+  return out
+}
+
 const authorFromUri = (uri: string): string => {
   // at://<did>/app.bsky.feed.post/<rkey>
   const match = uri.match(/^at:\/\/([^/]+)\//)
@@ -162,6 +182,7 @@ const toCandidateMeta = (row: {
   is_reply: number
   is_image: number
   is_video: number
+  langs: string
 }): CandidateMeta => ({
   uri: row.uri,
   author_did: row.author_did,
@@ -172,4 +193,5 @@ const toCandidateMeta = (row: {
   is_reply: row.is_reply === 1,
   is_image: row.is_image === 1,
   is_video: row.is_video === 1,
+  langs: row.langs ? row.langs.split(',').filter(Boolean) : [],
 })

src/ranker/popularity.ts

@@ -4,12 +4,15 @@ import { Ranker, ContentFilter } from './types'
 import { finalize } from './finalize'
 
 // Cold-start ranker for anonymous viewers and users with no likes yet: the
-// most-liked recent posts, still subject to time-decay and freshness.
+// most-liked recent posts, still subject to time-decay and freshness. When the
+// viewer's Accept-Language yields a non-empty `languages` allowlist, the feed is
+// biased toward those languages (see finalize); [] leaves it global.
 export class PopularityRanker implements Ranker {
   async rank(
     ctx: AppContext,
     _viewerDid: string | null,
     content: ContentFilter,
+    languages: string[] = [],
   ): Promise<string[]> {
     const cfg = ctx.cfg.ranking
     const cutoff = new Date(
@@ -33,7 +36,12 @@ export class PopularityRanker implements Ranker {
     const rawScores = new Map<string, number>()
     for (const row of rows.rows) rawScores.set(row.subject_uri, row.likes)
 
-    // No popularity penalty for cold start — popular posts should win.
-    return finalize(ctx, rawScores, { applyPopularityPenalty: false, content })
+    // No popularity penalty for cold start — popular posts should win. Bias
+    // toward the viewer's languages when known (undeclared posts still pass).
+    return finalize(ctx, rawScores, {
+      applyPopularityPenalty: false,
+      content,
+      languages,
+    })
   }
 }

src/ranker/types.ts

@@ -22,4 +22,7 @@ export type CandidateMeta = {
   is_reply: boolean
   is_image: boolean
   is_video: boolean
+  // normalized primary BCP-47 language subtags declared on the post (e.g.
+  // ['en', 'de']); empty when the post declares no language.
+  langs: string[]
 }