fix: shard sitemap.xml per locale

Even served statically, a single sitemap document is ~51MB / 17.5k URLs -- over Google's 50MB and 50k-URL per-file limits -- so crawlers reject it outright. Split it into one shard per locale via generateSitemaps(), each ~2MB / ~700 URLs, comfortably under both limits. Each shard still emits the full hreflang alternates block so cross-language reciprocity is preserved.

Next serves the shards at /sitemap/<locale>.xml and emits no index file, so robots.ts now lists every shard for crawler discovery in place of the old /sitemap.xml. A memoized traversal keeps all shards sharing a single content walk.

Verified against a full 25-locale build: all shards prerender static (initialRevalidateSeconds: false, no ISR), close with </urlset>, carry ~700 URLs each including /wallets/find-wallet/, and the largest is ~2MB.

Part 2 of #18014. Note: this removes the root /sitemap.xml; whatever is registered in Google Search Console should be updated to the shard URLs (robots.txt covers auto-discovery regardless).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Co-Authored-By: wackerow <54227730+wackerow@users.noreply.github.com>

Author: 3 people

app/robots.ts

@@ -1,6 +1,13 @@
 import type { MetadataRoute } from "next"
 
-import { IS_PRODUCTION_DEPLOY, SITE_URL } from "@/lib/constants"
+import { IS_PRODUCTION_DEPLOY, LOCALES_CODES, SITE_URL } from "@/lib/constants"
+
+// The sitemap is sharded per locale via generateSitemaps() (app/sitemap.ts).
+// Next serves each shard at /sitemap/<locale>.xml and emits no index file, so
+// every shard is listed here for crawler discovery.
+const sitemapShards = LOCALES_CODES.map(
+  (locale) => `${SITE_URL}/sitemap/${locale}.xml`
+)
 
 export default function robots(): MetadataRoute.Robots {
   if (!IS_PRODUCTION_DEPLOY) {
@@ -13,7 +20,7 @@ export default function robots(): MetadataRoute.Robots {
 
   return {
     rules: [{ userAgent: "*", allow: "/" }],
-    sitemap: [`${SITE_URL}/sitemap.xml`],
+    sitemap: sitemapShards,
     host: SITE_URL,
   }
 }

app/sitemap.ts

@@ -2,7 +2,7 @@ import type { MetadataRoute } from "next"
 
 import { getFullUrl, toLanguageTag } from "@/lib/utils/url"
 
-import { DEFAULT_LOCALE } from "@/lib/constants"
+import { DEFAULT_LOCALE, LOCALES_CODES } from "@/lib/constants"
 
 import { getAllPagesWithTranslations } from "@/lib/i18n/translationRegistry"
 
@@ -13,43 +13,56 @@ import { getAllPagesWithTranslations } from "@/lib/i18n/translationRegistry"
 // re-enters that broken path; freshness rides the deploy cadence instead.
 export const dynamic = "force-static"
 
-export default async function sitemap(): Promise<MetadataRoute.Sitemap> {
-  const pages = await getAllPagesWithTranslations()
+// One shard per locale. A single combined document is ~51MB / 17.5k URLs -- over
+// Google's 50MB per-file limit -- so it must be split. Next serves each shard at
+// /sitemap/<id>.xml and emits no index file, so app/robots.ts lists them all.
+export async function generateSitemaps() {
+  return LOCALES_CODES.map((locale) => ({ id: locale }))
+}
+
+// getAllPagesWithTranslations walks the whole content tree; memoize so the
+// per-locale shards share a single traversal instead of repeating it each time.
+let pagesPromise: ReturnType<typeof getAllPagesWithTranslations> | null = null
+const getPages = () => (pagesPromise ??= getAllPagesWithTranslations())
+
+export default async function sitemap({
+  id,
+}: {
+  id: Promise<string>
+}): Promise<MetadataRoute.Sitemap> {
+  const locale = await id
+  const pages = await getPages()
 
   const entries: MetadataRoute.Sitemap = []
   const seenUrls = new Set<string>()
 
   for (const { slug, translatedLocales } of pages) {
+    // This shard only carries URLs for its own locale; the full hreflang
+    // alternates block is still emitted so each URL cross-references every
+    // translated version.
+    if (!translatedLocales.includes(locale)) continue
+
     const normalizedSlug = slug.startsWith("/") ? slug : `/${slug}`
     const alternates =
       translatedLocales.length > 0
         ? {
             languages: {
               "x-default": getFullUrl(DEFAULT_LOCALE, normalizedSlug),
               ...Object.fromEntries(
-                translatedLocales.map((locale) => [
-                  toLanguageTag(locale),
-                  getFullUrl(locale, normalizedSlug),
+                translatedLocales.map((altLocale) => [
+                  toLanguageTag(altLocale),
+                  getFullUrl(altLocale, normalizedSlug),
                 ])
               ),
             },
           }
         : undefined
 
-    for (const locale of translatedLocales) {
-      const url = getFullUrl(locale, normalizedSlug)
-
-      if (seenUrls.has(url)) {
-        continue
-      }
-
-      seenUrls.add(url)
+    const url = getFullUrl(locale, normalizedSlug)
+    if (seenUrls.has(url)) continue
+    seenUrls.add(url)
 
-      entries.push({
-        url,
-        alternates,
-      })
-    }
+    entries.push({ url, alternates })
   }
 
   return entries