Fall back to root domain robots.txt when subdomain returns 404/403

If subdomain robots.txt is missing or inaccessible (404, 410, 403), fetch
root domain's robots.txt and use it; allow fetch when neither resolves.

Author: mattt

src/lib/external/fetch.ts

@@ -91,13 +91,13 @@ export async function fetchRobotsPolicy(
     },
   })
 
-  // Missing robots.txt is treated as no policy restrictions.
-  if (response.status === 404 || response.status === 410) {
-    return { kind: "allow-all" }
+  // Missing or inaccessible robots.txt — caller may try root domain or allow.
+  if (response.status === 404 || response.status === 410 || response.status === 403) {
+    return { kind: "not-found" }
   }
 
-  // Explicit access denial when robots cannot be read due to auth restrictions.
-  if (response.status === 401 || response.status === 403) {
+  // Explicit access denial when robots cannot be read due to auth.
+  if (response.status === 401) {
     return { kind: "deny-all" }
   }
 

src/lib/external/policy.ts

@@ -113,7 +113,10 @@ async function isAllowedByRobotsTxt(targetUrl: URL): Promise<boolean> {
   if (policy.kind === "deny-all") {
     return false
   }
-  return evaluateRobotsPolicy(policy.robotsText, targetUrl, EXTERNAL_DOC_USER_AGENT)
+  if (policy.kind === "rules") {
+    return evaluateRobotsPolicy(policy.robotsText, targetUrl, EXTERNAL_DOC_USER_AGENT)
+  }
+  return true
 }
 
 function evaluateRobotsPolicy(robotsText: string, targetUrl: URL, userAgent: string): boolean {
@@ -135,6 +138,20 @@ function parseHostList(rawList: string | undefined): Set<string> {
   )
 }
 
+function getRootOrigin(origin: string): string | null {
+  try {
+    const url = new URL(origin)
+    const labels = url.hostname.toLowerCase().split(".")
+    if (labels.length < 3) {
+      return null
+    }
+    const rootHost = labels.slice(-2).join(".")
+    return `${url.protocol}//${rootHost}`
+  } catch {
+    return null
+  }
+}
+
 async function getRobotsPolicy(origin: string): Promise<RobotsPolicyResult> {
   const now = Date.now()
   pruneExpiredRobotsPolicyEntries(now)
@@ -149,7 +166,23 @@ async function getRobotsPolicy(origin: string): Promise<RobotsPolicyResult> {
     return inFlight
   }
 
-  const request = fetchRobotsPolicy(origin, EXTERNAL_DOC_USER_AGENT)
+  const request = (async (): Promise<RobotsPolicyResult> => {
+    let policy = await fetchRobotsPolicy(origin, EXTERNAL_DOC_USER_AGENT)
+    if (policy.kind === "not-found") {
+      const rootOrigin = getRootOrigin(origin)
+      if (rootOrigin && rootOrigin !== origin) {
+        const rootPolicy = await fetchRobotsPolicy(rootOrigin, EXTERNAL_DOC_USER_AGENT)
+        if (rootPolicy.kind !== "not-found") {
+          policy = rootPolicy
+        } else {
+          policy = { kind: "allow-all" }
+        }
+      } else {
+        policy = { kind: "allow-all" }
+      }
+    }
+    return policy
+  })()
     .then((policy) => {
       robotsPolicyCache.set(origin, {
         expiresAt: Date.now() + ROBOTS_CACHE_TTL_MS,

src/lib/external/types.ts

@@ -1,6 +1,7 @@
 export type RobotsPolicyResult =
   | { kind: "allow-all" }
   | { kind: "deny-all" }
+  | { kind: "not-found" }
   | { kind: "rules"; robotsText: string }
 
 export interface ExternalPolicyEnv {

tests/external.test.ts

@@ -251,6 +251,93 @@ describe("External Swift-DocC support", () => {
     expect(global.fetch).toHaveBeenCalledTimes(1)
   })
 
+  it("should fall back to root domain robots.txt when subdomain robots.txt is 403 or 404", async () => {
+    // Real robots.txt from https://daily.co/robots.txt (subdomain reference-ios.daily.co returns 403, e.g. S3/CloudFront)
+    const dailyCoRobotsTxt = `# *
+User-agent: *
+Allow: /
+
+# Host
+Host: https://www.daily.co
+
+# Sitemaps
+Sitemap: https://www.daily.co/sitemap.xml
+Sitemap: https://www.daily.co/resources/sitemap.xml
+Sitemap: https://www.daily.co/partners/sitemap.xml
+Sitemap: https://www.daily.co/videosaurus/sitemap.xml
+Sitemap: https://www.daily.co/blog/sitemap.xml
+Sitemap: https://docs.daily.co/sitemap.xml
+`
+
+    global.fetch = vi.fn().mockImplementation((url: string | URL) => {
+      const u = typeof url === "string" ? url : url.toString()
+      if (u === "https://reference-ios.daily.co/robots.txt") {
+        return Promise.resolve(
+          new Response(null, {
+            status: 403,
+            headers: { "content-type": "application/xml" },
+          }),
+        )
+      }
+      if (u === "https://daily.co/robots.txt") {
+        return Promise.resolve(
+          new Response(dailyCoRobotsTxt, {
+            status: 200,
+            headers: { "Content-Type": "text/plain" },
+          }),
+        )
+      }
+      return Promise.reject(new Error(`Unexpected fetch: ${u}`))
+    })
+
+    await expect(
+      assertExternalDocumentationAccess(
+        new URL("https://reference-ios.daily.co/documentation/some/module"),
+        {},
+      ),
+    ).resolves.toBeUndefined()
+
+    expect(global.fetch).toHaveBeenNthCalledWith(
+      1,
+      "https://reference-ios.daily.co/robots.txt",
+      expect.objectContaining({
+        headers: expect.objectContaining({ "User-Agent": EXTERNAL_DOC_USER_AGENT }),
+      }),
+    )
+    expect(global.fetch).toHaveBeenNthCalledWith(
+      2,
+      "https://daily.co/robots.txt",
+      expect.objectContaining({
+        headers: expect.objectContaining({ "User-Agent": EXTERNAL_DOC_USER_AGENT }),
+      }),
+    )
+    expect(global.fetch).toHaveBeenCalledTimes(2)
+  })
+
+  it("should allow fetch when both subdomain and root domain robots.txt are 404", async () => {
+    global.fetch = vi.fn().mockImplementation((url: string | URL) => {
+      const u = typeof url === "string" ? url : url.toString()
+      if (u === "https://docs.example.org/robots.txt" || u === "https://example.org/robots.txt") {
+        return Promise.resolve(new Response(null, { status: 404 }))
+      }
+      return Promise.reject(new Error(`Unexpected fetch: ${u}`))
+    })
+
+    await expect(
+      assertExternalDocumentationAccess(
+        new URL("https://docs.example.org/documentation/some/module"),
+        {},
+      ),
+    ).resolves.toBeUndefined()
+
+    expect(global.fetch).toHaveBeenCalledWith(
+      "https://docs.example.org/robots.txt",
+      expect.any(Object),
+    )
+    expect(global.fetch).toHaveBeenCalledWith("https://example.org/robots.txt", expect.any(Object))
+    expect(global.fetch).toHaveBeenCalledTimes(2)
+  })
+
   it("should build and fetch external DocC JSON", async () => {
     global.fetch = vi
       .fn()