feat: add dynamic sitemap route map

src/sitemap.ts

@@ -0,0 +1,114 @@
+import { buildDeno2AppSlug } from './utils/build-deno-url'
+
+const GITHUB_API_BASE = 'https://api.github.com'
+const ORG = 'ubiquity'
+const CACHE_HEADERS = {
+  'Cache-Control': 'public, max-age=300',
+}
+
+type GitHubRepo = Readonly<{
+  name: string
+  archived?: boolean
+  disabled?: boolean
+}>
+
+export type RouteMapEntry = Readonly<{
+  type: 'app' | 'plugin'
+  name: string
+  host: string
+  url: string
+  upstream: string
+}>
+
+export async function handleRouteMap(requestUrl: URL): Promise<Response | null> {
+  if (requestUrl.pathname === '/sitemap.xml') {
+    const entries = await buildRouteMap(requestUrl.origin)
+    return new Response(toSitemapXml(entries), {
+      headers: { ...CACHE_HEADERS, 'Content-Type': 'application/xml; charset=utf-8' },
+    })
+  }
+
+  if (requestUrl.pathname === '/routes.json' || requestUrl.pathname === '/sitemap.json') {
+    const entries = await buildRouteMap(requestUrl.origin)
+    return new Response(JSON.stringify({ generatedAt: new Date().toISOString(), entries }, null, 2), {
+      headers: { ...CACHE_HEADERS, 'Content-Type': 'application/json; charset=utf-8' },
+    })
+  }
+
+  return null
+}
+
+export async function buildRouteMap(origin = 'https://ubq.fi'): Promise<RouteMapEntry[]> {
+  const repos = await fetchOrgRepos()
+  const apps = repos
+    .filter((repo) => repo.name === 'ubq.fi' || repo.name.endsWith('.ubq.fi'))
+    .map((repo) => {
+      const subdomain = repo.name === 'ubq.fi' ? '' : repo.name.replace(/\.ubq\.fi$/, '')
+      const host = subdomain ? `${subdomain}.ubq.fi` : 'ubq.fi'
+      return {
+        type: 'app' as const,
+        name: repo.name,
+        host,
+        url: `${originForHost(origin, host)}/`,
+        upstream: `https://${buildDeno2AppSlug(subdomain)}.ubiquity-dao.deno.net/`,
+      }
+    })
+
+  const plugins = repos
+    .filter((repo) => repo.name.startsWith('ubiquity-os-') || repo.name.startsWith('plugin-'))
+    .map((repo) => {
+      const plugin = repo.name.replace(/^ubiquity-os-/, '').replace(/^plugin-/, '')
+      const host = `os-${plugin}.ubq.fi`
+      return {
+        type: 'plugin' as const,
+        name: repo.name,
+        host,
+        url: `${originForHost(origin, host)}/`,
+        upstream: `https://${plugin}-main.deno.dev/`,
+      }
+    })
+
+  return [...apps, ...plugins].sort((a, b) => a.host.localeCompare(b.host))
+}
+
+async function fetchOrgRepos(): Promise<GitHubRepo[]> {
+  const repos: GitHubRepo[] = []
+  for (let page = 1; page <= 10; page++) {
+    const response = await fetch(`${GITHUB_API_BASE}/orgs/${ORG}/repos?per_page=100&page=${page}`, {
+      headers: {
+        Accept: 'application/vnd.github+json',
+        'User-Agent': 'ubq-fi-router',
+      },
+    })
+    if (!response.ok) break
+    const batch = await response.json() as GitHubRepo[]
+    repos.push(...batch.filter((repo) => !repo.archived && !repo.disabled))
+    if (batch.length < 100) break
+  }
+  return repos
+}
+
+function originForHost(origin: string, host: string): string {
+  const url = new URL(origin)
+  url.hostname = host
+  url.pathname = ''
+  url.search = ''
+  url.hash = ''
+  return url.toString().replace(/\/$/, '')
+}
+
+function toSitemapXml(entries: RouteMapEntry[]): string {
+  const urls = entries
+    .map((entry) => `  <url>\n    <loc>${escapeXml(entry.url)}</loc>\n  </url>`)
+    .join('\n')
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`
+}
+
+function escapeXml(value: string): string {
+  return value
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+}

src/worker.ts

@@ -12,6 +12,7 @@ import {
   resolveDenoUrl,
 } from './utils/build-deno-url'
 import { buildPluginUrl } from './utils/build-plugin-url'
+import { handleRouteMap } from './sitemap'
 
 declare const __UOS_ROUTER_REVISION__: string | undefined
 
@@ -61,6 +62,11 @@ export default {
   async fetch(request: Request, env: Env): Promise<Response> {
     const url = new URL(request.url)
 
+    const routeMapResponse = await handleRouteMap(url)
+    if (routeMapResponse) {
+      return withRouterRevision(routeMapResponse)
+    }
+
     if (url.pathname === '/__health') {
       if (shouldLog('health', request, url, env)) {
         try {

tests/sitemap.test.ts

@@ -0,0 +1,55 @@
+import { describe, expect, test } from 'bun:test'
+import worker, { type Env } from '../src/worker'
+import { buildRouteMap } from '../src/sitemap'
+
+const githubResponse = [
+  { name: 'ubq.fi' },
+  { name: 'pay.ubq.fi' },
+  { name: 'ubiquity-os-kernel' },
+  { name: 'old.ubq.fi', archived: true },
+]
+
+describe('dynamic route map', () => {
+  test('builds JSON route map from active GitHub repos', async () => {
+    globalThis.fetch = (async () => Response.json(githubResponse)) as unknown as typeof fetch
+
+    const entries = await buildRouteMap('https://ubq.fi')
+
+    expect(entries).toEqual([
+      {
+        type: 'plugin',
+        name: 'ubiquity-os-kernel',
+        host: 'os-kernel.ubq.fi',
+        url: 'https://os-kernel.ubq.fi/',
+        upstream: 'https://kernel-main.deno.dev/',
+      },
+      {
+        type: 'app',
+        name: 'pay.ubq.fi',
+        host: 'pay.ubq.fi',
+        url: 'https://pay.ubq.fi/',
+        upstream: 'https://pay-ubq-fi.ubiquity-dao.deno.net/',
+      },
+      {
+        type: 'app',
+        name: 'ubq.fi',
+        host: 'ubq.fi',
+        url: 'https://ubq.fi/',
+        upstream: 'https://ubq-fi.ubiquity-dao.deno.net/',
+      },
+    ])
+  })
+
+  test('serves sitemap xml through the worker', async () => {
+    globalThis.fetch = (async () => Response.json(githubResponse)) as unknown as typeof fetch
+
+    const res = await worker.fetch(new Request('https://ubq.fi/sitemap.xml'), {} as Env)
+    const body = await res.text()
+
+    expect(res.status).toBe(200)
+    expect(res.headers.get('content-type')).toContain('application/xml')
+    expect(res.headers.get('x-uos-router-revision')).toBe('local')
+    expect(body).toContain('<loc>https://pay.ubq.fi/</loc>')
+    expect(body).toContain('<loc>https://os-kernel.ubq.fi/</loc>')
+  })
+})