feat: Make robots.txt environment-aware using DEPLOYMENT_HOST

- Add generate-robots-txt.js script that uses DEPLOYMENT_HOST env var
- Fallback to atmos.tools (production) when DEPLOYMENT_HOST is not set
- Add robots.txt to static/.gitignore since it's now generated at build time
- Run generation in prebuild step alongside themes.json copy

This ensures non-production deployments (staging, preview) have the correct
sitemap URL in robots.txt, matching the same pattern used in algolia/reindex.sh.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Haiku 4.5 <noreply@anthropic.com>

Author: osterman

website/package.json

@@ -4,7 +4,7 @@
   "license": "Apache-2.0",
   "scripts": {
     "docusaurus": "docusaurus",
-    "prebuild": "cp ../pkg/ui/theme/themes.json static/themes.json",
+    "prebuild": "cp ../pkg/ui/theme/themes.json static/themes.json && node scripts/generate-robots-txt.js",
     "start": "BROWSER=open docusaurus start",
     "build": "docusaurus build",
     "postbuild": "cp build/llms.txt static/llms.txt && cp build/llms-full.txt static/llms-full.txt",

website/scripts/generate-robots-txt.js

@@ -0,0 +1,26 @@
+#!/usr/bin/env node
+
+/**
+ * Generates robots.txt with environment-aware sitemap URL.
+ * Uses DEPLOYMENT_HOST env var with fallback to atmos.tools (production).
+ */
+
+const fs = require('fs');
+const path = require('path');
+
+const DEPLOYMENT_HOST = process.env.DEPLOYMENT_HOST || 'atmos.tools';
+
+const robotsTxt = `# Algolia-Crawler-Verif: 10F61B92D9EB1214
+
+# Allow all crawlers to index all content
+User-agent: *
+Allow: /
+
+# Sitemap location
+Sitemap: https://${DEPLOYMENT_HOST}/sitemap.xml
+`;
+
+const outputPath = path.join(__dirname, '..', 'static', 'robots.txt');
+fs.writeFileSync(outputPath, robotsTxt);
+
+console.log(`Generated robots.txt with sitemap URL: https://${DEPLOYMENT_HOST}/sitemap.xml`);

website/static/.gitignore

@@ -1,2 +1,3 @@
 glossary.json
+robots.txt
 themes.json