fix: fix batching mechanism

Author: danielroe

app/app.vue

@@ -9,7 +9,13 @@ const { data: clusters, refresh, status } = useFetch('/api/clusters/nuxt/nuxt',
 
 onMounted(async () => {
   if ('startViewTransition' in document) {
-    document.startViewTransition(() => refresh())
+    let finishTransition: () => void
+    const promise = new Promise<void>((resolve) => {
+      finishTransition = resolve
+    })
+    watch(clusters, () => document.startViewTransition(() => promise), { flush: 'pre', once: true })
+    watch(clusters, () => nextTick(finishTransition), { flush: 'post', once: true })
+    refresh()
   }
 })
 
@@ -152,6 +158,8 @@ async function updateSearch() {}
                   :datetime="issue.updated_at"
                   relative
                 />
+                &middot;
+                {{ Math.floor(issue.avgSimilarity * 100) }}% similar
               </span>
               <div class="flex flex-row gap-1 items-baseline flex-wrap md:flex-nowrap">
                 <span

server/api/clusters/[...repo].ts

@@ -1,13 +1,13 @@
 import { clusterEmbeddings } from '../../utils/cluster'
 import { getEmbeddingsForIssue } from '../../utils/embeddings'
-import type { Issue } from '../../utils/github'
 
 const linkedRepos: Record<string, string[]> = {
   'danielroe/beasties': ['GoogleChromeLabs/critters'],
+  'nitrojs/nitro': ['unjs/h3', 'unjs/c12', 'unjs/unenv', 'unjs/ofetch'],
   'nuxt/nuxt': ['vuejs/core', 'vitejs/vite', 'nitrojs/nitro'],
 }
 
-const allowedRepos = ['nuxt/nuxt', 'vuejs/vue', 'vitejs/vite', 'nitrojs/nitro', 'danielroe/beasties']
+const allowedRepos = ['nuxt/nuxt', 'vuejs/core', 'vitejs/vite', 'nitrojs/nitro', 'danielroe/beasties', 'unjs/h3', 'unjs/c12', 'unjs/unenv', 'unjs/ofetch']
 
 export default defineCachedEventHandler(async (event) => {
   const [owner, repo] = getRouterParam(event, 'repo')?.split('/') || []
@@ -28,27 +28,30 @@ export default defineCachedEventHandler(async (event) => {
   }
 
   const repos = [source, ...linkedRepos[source] || []]
-  const issues = await Promise.all(repos.map(async repo => event.$fetch<Issue[]>(`/api/issues/${repo}`)))
+  const issues = await Promise.all(repos.map(async repo => $fetch(`/api/issues/${repo}`)))
     .then(r => r.flat())
 
   console.log('fetched', issues.length, 'issues')
 
+  const backlog = [...issues]
   const embeddings: number[][] = []
 
   do {
-    const batch = issues.splice(0, 100)
+    const batch = backlog.splice(0, 100)
     embeddings.push(...await Promise.all(batch.map(async issue => getEmbeddingsForIssue(event, issue))))
-  } while (issues.length)
+  } while (backlog.length)
 
+  console.log('generated', embeddings.length, 'embeddings')
   const clusters = clusterEmbeddings(issues, embeddings)
+  console.log('generated', clusters.length, 'clusters')
   return clusters.filter((cluster) => {
     return cluster.some(issue => issue.repository?.owner?.name === owner && issue.repository?.name === repo)
   })
 }, {
   swr: true,
   getKey(event) {
     const [owner, repo] = getRouterParam(event, 'repo')?.split('/') || []
-    return `clusters:${owner}:${repo}`
+    return `clusters:${owner}:${repo}`.toLowerCase()
   },
   maxAge: 60 * 60 * 1000,
   staleMaxAge: 60 * 60 * 1000,

server/api/issues/[...repo].ts

@@ -5,7 +5,7 @@ import { getLabels, type Issue } from '../../utils/github'
 const labelsToExclude = ['documentation', 'invalid', 'enhancement']
 const knownBots = new Set(['renovate', 'renovate[bot]'])
 
-const allowedRepos = ['nuxt/nuxt', 'vuejs/vue', 'vitejs/vite', 'nitrojs/nitro', 'danielroe/beasties']
+const allowedRepos = ['nuxt/nuxt', 'vuejs/core', 'vitejs/vite', 'nitrojs/nitro', 'danielroe/beasties', 'unjs/h3', 'unjs/c12', 'unjs/unenv', 'unjs/ofetch']
 
 export default defineCachedEventHandler(async (event) => {
   const [owner, repo] = getRouterParam(event, 'repo')?.split('/') || []
@@ -23,7 +23,8 @@ export default defineCachedEventHandler(async (event) => {
     })
   }
 
-  const octokit = new Octokit({ auth: useRuntimeConfig(event).github.token })
+  const options = owner !== 'unjs' ? { auth: useRuntimeConfig(event).github.token } : {}
+  const octokit = new Octokit(options)
 
   // TODO: date/state filters?
   const issues = await octokit.paginate(octokit.issues.listForRepo, {
@@ -42,7 +43,7 @@ export default defineCachedEventHandler(async (event) => {
   swr: true,
   getKey(event) {
     const [owner, repo] = getRouterParam(event, 'repo')?.split('/') || []
-    return `issues:${owner}:${repo}`
+    return `issues:${owner}:${repo}`.toLowerCase()
   },
   maxAge: 60 * 60 * 1000,
   staleMaxAge: 60 * 60 * 1000,

server/utils/cluster.ts

@@ -26,16 +26,20 @@ export function clusterEmbeddings(issues: Issue[], embeddings: number[][]) {
 
   for (const cluster in clusteredIssues) {
     const chunk = clusteredIssues[cluster]!
-    const sortedChunk = chunk.map((issue) => {
+    let sortedChunk = chunk.map((issue) => {
       let totalSimilarity = 0
       for (const other of chunk) {
         if (issue === other) {
           continue
         }
         totalSimilarity += similarity.cosine(embeddings[issues.indexOf(issue)], embeddings[issues.indexOf(other)])
       }
-      return { ...issue, avgSimilarity: totalSimilarity / (chunk.length - 1) }
-    }).filter(i => i.avgSimilarity > 0.75).sort((a, b) => b.avgSimilarity - a.avgSimilarity)
+      return { ...issue, avgSimilarity: Math.floor(100 * totalSimilarity / (chunk.length - 1)) / 100 }
+    })
+
+    const similarityThreshold = 0.75
+
+    sortedChunk = sortedChunk.filter(i => i.avgSimilarity >= similarityThreshold).sort((a, b) => b.avgSimilarity - a.avgSimilarity)
 
     if (sortedChunk.length > 1) {
       sortedClusters.push(sortedChunk)