tweak robots.txt file

Author: sm47916

robots.txt

@@ -1,3 +1,43 @@
-User-Agent: *
+# Block aggressive AI training crawlers
+User-agent: meta-externalagent
+User-agent: GPTBot
+User-agent: Google-Extended
+User-agent: CCBot
+User-agent: anthropic-ai
+User-agent: Claude-Web
+User-agent: Bytespider
+User-agent: PerplexityBot
+User-agent: YouBot
+User-agent: Amazonbot
+User-agent: cohere-ai
+Disallow: /
+
+User-agent: *
 Allow: /
 
+# Allow OG image so social crawlers (Facebook, LinkedIn, Slack, etc.) can render link previews
+Allow: /images/district0x-1200x630.png
+
+# Block PDF and media asset directories
+Disallow: /docs/
+Disallow: /images/
+Disallow: /favicons/
+
+Sitemap: https://district0x.io/sitemap.xml
+
+# Block media files by extension (catch-all for assets outside the above dirs)
+Disallow: /*.pdf$
+Disallow: /*.png$
+Disallow: /*.jpg$
+Disallow: /*.jpeg$
+Disallow: /*.gif$
+Disallow: /*.webp$
+Disallow: /*.svg$
+Disallow: /*.ico$
+Disallow: /*.mp4$
+Disallow: /*.webm$
+Disallow: /*.mp3$
+Disallow: /*.wav$
+Disallow: /*.woff$
+Disallow: /*.woff2$
+Disallow: /*.ttf$