Merge pull request #238 from onflow/brian-doyle/add-robots.txt

Add robots.txt

Author: briandoyle81

static/robots.txt

@@ -0,0 +1,37 @@
+# algolia: 98E1096D4FD67E70
+
+# Allow search engines
+User-agent: Googlebot
+Allow: /
+
+User-agent: Bingbot
+Allow: /
+
+User-agent: DuckDuckBot
+Allow: /
+
+User-agent: Baiduspider
+Allow: /
+
+# Allow LLM bots
+User-agent: GPTBot
+Allow: /
+
+User-agent: CCBot
+Allow: /
+
+User-agent: anthropic-ai
+Allow: /
+
+User-agent: ClaudeBot
+Allow: /
+
+User-agent: PerplexityBot
+Allow: /
+
+# Block all other bots by default
+User-agent: *
+Disallow: /
+
+# Sitemap
+Sitemap: https://cadence-lang.org/sitemap.xml