feat: allow manually indexing incomplete repos

Author: danielroe

nuxt.config.ts

@@ -27,6 +27,7 @@ export default defineNuxtConfig({
       },
     },
     github: {
+      token: '',
       appId: '',
       privateKey: '',
     },
@@ -40,6 +41,11 @@ export default defineNuxtConfig({
     typedPages: true,
   },
   compatibilityDate: '2024-04-03',
+  nitro: {
+    experimental: {
+      tasks: true,
+    },
+  },
   hub: {
     ai: true,
     analytics: true,

server/api/clusters/[owner]/[repo].get.ts

@@ -23,7 +23,7 @@ export default defineCachedCorsEventHandler(async (event) => {
     })
 
   const clusters = clusterEmbeddings(issues, embeddings)
-  console.log('generated', clusters.length, 'clusters')
+  console.log('generated', clusters.length, 'clusters from', issues.length, embeddings.length)
 
   return clusters
     .map(cluster => cluster.map(i => ({

server/routes/github/webhook.post.ts

@@ -93,35 +93,46 @@ async function addRepos(event: H3Event, installation: Installation | Installatio
       continue
     }
 
-    console.log('starting to index', `${repo.full_name}`)
+    await indexRepo(octokit, repo)
 
     const [owner, name] = repo.full_name.split('/')
+    event.waitUntil(setMetadataForRepo(owner!, name!, { ...repo, indexed: true }))
+  }
+}
 
-    const promises: Array<Promise<unknown>> = []
-
-    await octokit.paginate(octokit.rest.issues.listForRepo, {
-      owner: owner!,
-      repo: name!,
-      state: 'open',
-      per_page: 100,
-    }, (response) => {
-      console.log(response.headers)
-      for (const issue of response.data) {
-        promises.push(indexIssue(issue, { owner: { login: owner! }, name: name! }))
-      }
-      return []
-    })
+export async function indexRepo(octokit: Octokit, repo: InstallationRepo) {
+  if (repo.private) {
+    return
+  }
 
-    await Promise.allSettled(promises).then((r) => {
-      if (r.some(p => p.status === 'rejected')) {
-        console.error('Failed to fetch some issues from', `${owner}/${name}`)
-      }
-    })
+  console.log('starting to index', `${repo.full_name}`)
 
-    console.log('added', promises.length - 1, 'issues from', `${owner}/${name}`, 'to the index')
+  const [owner, name] = repo.full_name.split('/')
 
-    event.waitUntil(setMetadataForRepo(owner!, name!, { ...repo, indexed: true }))
-  }
+  const promises: Array<Promise<unknown>> = []
+
+  await octokit.paginate(octokit.rest.issues.listForRepo, {
+    owner: owner!,
+    repo: name!,
+    state: 'open',
+    per_page: 100,
+  }, (response) => {
+    for (const issue of response.data) {
+      promises.push(indexIssue(issue, { owner: { login: owner! }, name: name! }))
+    }
+    if (parseInt(response.headers['x-ratelimit-remaining'] || '999') < 100) {
+      console.info(parseInt(response.headers['x-ratelimit-remaining']!), 'requests remaining')
+    }
+    return []
+  })
+
+  await Promise.allSettled(promises).then((r) => {
+    if (r.some(p => p.status === 'rejected')) {
+      console.error('Failed to fetch some issues from', `${owner}/${name}`)
+    }
+  })
+
+  console.log('added', promises.length, 'issues from', `${owner}/${name}`, 'to the index')
 }
 
 async function deleteRepo(event: H3Event, repo: InstallationRepo) {

server/tasks/index-repo.ts

@@ -0,0 +1,39 @@
+import { Octokit } from '@octokit/rest'
+import { defineTask } from 'nitropack/runtime'
+import { indexRepo } from '~~/server/routes/github/webhook.post'
+
+export default defineTask({
+  meta: {
+    name: 'index-repo',
+    description: 'Index repositories',
+  },
+  async run() {
+    const octokit = new Octokit({ auth: useRuntimeConfig().github.token })
+
+    const repos = await $fetch('/api/repos')
+    const indexed: string[] = []
+    let count = 1
+    for (const repo of repos) {
+      if (repo.indexed) continue
+
+      try {
+        const [owner, name] = repo.repo.split('/')
+        const meta = await getMetadataForRepo(owner!, name!)
+        if (!meta) continue
+
+        await indexRepo(octokit, meta)
+        indexed.push(repo.repo)
+        await setMetadataForRepo(owner!, name!, { ...meta, indexed: true })
+      }
+      catch (e) {
+        console.error(e)
+      }
+
+      if (count++ > 30) {
+        break
+      }
+    }
+
+    return indexed
+  },
+})