Add support for externally-hosted DocC websites

Author: mattt

README.md

@@ -26,6 +26,41 @@ https://sosumi.ai/documentation/swift/array
 This works for all API reference docs, 
 as well as Apple's [Human Interface Guidelines](https://developer.apple.com/design/human-interface-guidelines/) (HIG).
 
+### External Swift-DocC sites
+
+Sosumi can also proxy public non-Apple Swift-DocC pages using:
+
+```
+https://sosumi.ai/external/https://<host>/documentation/<path>
+```
+
+Example:
+
+```
+https://sosumi.ai/external/https://reference-ios.daily.co/documentation/daily
+```
+
+Under the hood, Sosumi resolves these to the source DocC JSON endpoint
+(`https://<host>/data/documentation/<path>.json`) and renders Markdown.
+
+#### External access controls (self-hosting)
+
+You can restrict which external hosts are allowed:
+
+- `EXTERNAL_DOC_HOST_ALLOWLIST` - optional newline-delimited host allowlist
+- `EXTERNAL_DOC_HOST_BLOCKLIST` - optional newline-delimited host blocklist
+
+If an allowlist is set, only listed hosts are permitted.
+Blocklist entries always deny access.
+
+Sosumi also checks `robots.txt` for external hosts before fetching content.
+Site owners can block Sosumi by disallowing:
+
+- User-agent: `sosumi-ai` (full UA: `sosumi-ai/1.0 (+https://sosumi.ai/#bot)`)
+- or wildcard `*`
+
+See `/bot` for the crawler policy and contact details.
+
 ### MCP Integration
 
 Sosumi's MCP server supports Streamable HTTP and Server-Sent Events (SSE) transport. 
@@ -175,6 +210,9 @@ It does not crawl, spider, or bulk download;
 it does not attempt to bypass authentication or security;
 and it implements rate limiting to avoid imposing unreasonable load.
 
+For external Swift-DocC hosts, access can be denied by `robots.txt`
+and opt-out response directives such as `X-Robots-Tag: noai`.
+
 Content is fetched transiently and may be cached briefly to improve performance.
 No permanent archives are maintained.
 All copyrights and other rights in the underlying content remain with Apple Inc.

public/index.html

@@ -594,11 +594,33 @@ <h2>Examples</h2>
                     <li><a href="/documentation/swift">Swift</a></li>
                     <li><a href="/documentation/swiftui">SwiftUI</a></li>
                     <li><a href="/design/human-interface-guidelines/">Human Interface Guidelines</a></li>
+                    <li><a
+                            href="/external/https://apple.github.io/swift-argument-parser/documentation/argumentparser/">Swift
+                            Argument Parser (external)</a></li>
                 </ul>
             </section>
         </section>
 
 
+        <section id="bot">
+            <header>
+                <h2>Bot & Crawling Policy</h2>
+                <p>
+                    Automated fetches with the user agent
+                    <code>sosumi-ai/1.0 (+https://sosumi.ai/#bot)</code>.
+                </p>
+                <p>
+                    For external Swift-DocC hosts, sosumi.ai honors
+                    <code>robots.txt</code> rules and opt-out response directives such as
+                    <code>X-Robots-Tag: noai</code>.
+                </p>
+                <p>
+                    Questions or issues: <a href="mailto:info@sosumi.ai">info@sosumi.ai</a>
+                </p>
+            </header>
+        </section>
+
+
 
         <section id="mcp">
             <header>

src/index.ts

@@ -6,6 +6,13 @@ import { HTTPException } from "hono/http-exception"
 import { trimTrailingSlash } from "hono/trailing-slash"
 
 import { NotFoundError } from "./lib/fetch"
+import {
+  assertExternalDocumentationAccess,
+  extractExternalDocumentationBasePath,
+  ExternalAccessError,
+  fetchExternalDocCJSON,
+  validateExternalDocumentationUrl,
+} from "./lib/external"
 import {
   fetchHIGPageData,
   fetchHIGTableOfContents,
@@ -19,6 +26,8 @@ import { generateAppleDocUrl, isValidAppleDocUrl, normalizeDocumentationPath } f
 interface Env {
   ASSETS: Fetcher
   NODE_ENV: string
+  EXTERNAL_DOC_HOST_ALLOWLIST?: string
+  EXTERNAL_DOC_HOST_BLOCKLIST?: string
 }
 
 const app = new Hono<{ Bindings: Env }>()
@@ -63,6 +72,8 @@ app.all("/mcp", async (c) => {
   return transport.handleRequest(c)
 })
 
+app.get("/bot", (c) => c.redirect("/#bot", 302))
+
 app.get("/documentation/*", async (c) => {
   const path = c.req.path
 
@@ -135,6 +146,50 @@ This service only works with Apple Developer documentation URLs:
   })
 })
 
+app.get("/external/*", async (c) => {
+  const path = c.req.path
+  const rawTarget = decodeURIComponent(path.replace("/external/", ""))
+  const targetUrl = validateExternalDocumentationUrl(rawTarget)
+
+  await assertExternalDocumentationAccess(targetUrl, c.env)
+  const jsonData = await fetchExternalDocCJSON(targetUrl)
+  const externalBasePath = extractExternalDocumentationBasePath(targetUrl)
+  const markdown = await renderFromJSON(jsonData, targetUrl.toString(), {
+    externalOrigin: `${targetUrl.origin}${externalBasePath}`,
+  })
+
+  if (!markdown || markdown.trim().length < 100) {
+    throw new HTTPException(502, {
+      message:
+        "The external documentation page loaded but contained insufficient content. This may be a temporary issue with the page.",
+    })
+  }
+
+  const headers = {
+    "Content-Type": "text/markdown; charset=utf-8",
+    "Content-Location": targetUrl.toString(),
+    "Cache-Control": "public, max-age=3600, s-maxage=86400",
+    ETag: `"${Buffer.from(markdown).toString("base64").slice(0, 16)}"`,
+    "Last-Modified": new Date().toUTCString(),
+  }
+
+  if (c.req.header("Accept")?.includes("application/json")) {
+    return c.json(
+      {
+        url: targetUrl.toString(),
+        content: markdown,
+      },
+      200,
+      { ...headers, "Content-Type": "application/json; charset=utf-8" },
+    )
+  }
+
+  return c.text(markdown, 200, {
+    ...headers,
+    "Content-Type": "text/markdown; charset=utf-8",
+  })
+})
+
 app.get("/design/human-interface-guidelines", async (c) => {
   // Handle the table of contents for HIG
   const tocData = await fetchHIGTableOfContents()
@@ -275,6 +330,36 @@ The requested Apple Developer documentation page does not exist.
     )
   }
 
+  if (err instanceof ExternalAccessError) {
+    const accept = c.req.header("Accept")
+    if (accept?.includes("application/json")) {
+      return c.json(
+        {
+          error: "External documentation access denied",
+          message: err.message,
+        },
+        { status: err.status as 400 | 403 },
+      )
+    }
+
+    return c.text(
+      `# External Documentation Access Denied
+
+${err.message}
+
+## Opt-out controls supported
+
+- \`robots.txt\` disallow for \`sosumi-ai-bot\` (or \`*\`)
+- \`X-Robots-Tag\` response directives such as \`noai\`, \`noimageai\`, \`noindex\`
+- Local operator host controls: \`EXTERNAL_DOC_HOST_ALLOWLIST\`, \`EXTERNAL_DOC_HOST_BLOCKLIST\`
+
+---
+*[sosumi.ai](https://sosumi.ai) - Making docs AI-readable*`,
+      err.status as 400 | 403,
+      { "Content-Type": "text/markdown; charset=utf-8" },
+    )
+  }
+
   // Handle unexpected errors
   const accept = c.req.header("Accept")
   if (accept?.includes("application/json")) {

src/lib/external/fetch.ts

@@ -0,0 +1,76 @@
+import { NotFoundError } from "../fetch"
+import type { AppleDocJSON } from "../types"
+import { EXTERNAL_DOC_USER_AGENT, ExternalAccessError } from "./policy"
+
+const RESTRICTIVE_X_ROBOTS_TAGS = ["none", "noindex", "noai", "noimageai"] as const
+
+export function extractExternalDocumentationBasePath(sourceUrl: URL): string {
+  const normalizedPath = sourceUrl.pathname.replace(/\/+$/, "")
+  const match = normalizedPath.match(/^(.*?)(\/documentation(?:\/.*)?)$/)
+  if (!match) {
+    throw new ExternalAccessError(
+      "External URL must point to a Swift-DocC documentation path.",
+      400,
+    )
+  }
+
+  return match[1]
+}
+
+export function buildExternalDocCJsonUrl(sourceUrl: URL): URL {
+  const hostBasePath = extractExternalDocumentationBasePath(sourceUrl)
+  const documentationPath = sourceUrl.pathname.replace(/\/+$/, "").slice(hostBasePath.length)
+  const jsonPath = documentationPath.endsWith(".json")
+    ? documentationPath
+    : `${documentationPath}.json`
+  return new URL(`${hostBasePath}/data${jsonPath}`, sourceUrl.origin)
+}
+
+export async function fetchExternalDocCJSON(sourceUrl: URL): Promise<AppleDocJSON> {
+  const jsonUrl = buildExternalDocCJsonUrl(sourceUrl)
+  const response = await fetch(jsonUrl.toString(), {
+    headers: {
+      "User-Agent": EXTERNAL_DOC_USER_AGENT,
+      Accept: "application/json",
+    },
+  })
+
+  const xRobotsTag = response.headers.get("x-robots-tag")
+  if (containsRestrictiveXRobotsTag(xRobotsTag)) {
+    throw new ExternalAccessError(
+      "External host denied AI/doc access via X-Robots-Tag response header.",
+      403,
+    )
+  }
+
+  if (!response.ok) {
+    if (response.status === 404) {
+      throw new NotFoundError(`External documentation page not found at ${jsonUrl.toString()}`)
+    }
+
+    throw new Error(`Failed to fetch external DocC JSON: ${response.status} ${response.statusText}`)
+  }
+
+  return (await response.json()) as AppleDocJSON
+}
+
+function containsRestrictiveXRobotsTag(headerValue: string | null): boolean {
+  if (!headerValue) {
+    return false
+  }
+
+  const tokenSet = new Set(
+    headerValue
+      .toLowerCase()
+      .split(",")
+      .map((token) => token.trim())
+      .filter(Boolean),
+  )
+
+  for (const token of RESTRICTIVE_X_ROBOTS_TAGS) {
+    if (tokenSet.has(token)) {
+      return true
+    }
+  }
+  return false
+}

src/lib/external/index.ts

@@ -0,0 +1,2 @@
+export * from "./fetch"
+export * from "./policy"