feat: serve robots.txt disallowing all crawlers

Personal read-it-later instance, not meant to be indexed.

Author: abdusco

main.go

@@ -87,6 +87,7 @@ func main() {
 	e.GET("/articles/:id/assets/*", s.handleArticleAsset)
 	e.POST("/articles/epub", s.handleEPUB)
 	e.GET("/readone.user.js", serveUserscript)
+	e.GET("/robots.txt", handleRobots)
 	e.StaticFS("/assets", assetsFS)
 
 	// The userscript runs on whatever article page the user is reading, so
@@ -108,6 +109,12 @@ func main() {
 	e.Logger.Fatal(e.Start(":" + port))
 }
 
+// This is a personal read-it-later tool, not something meant to be indexed —
+// block every crawler rather than trying to enumerate specific ones.
+func handleRobots(c echo.Context) error {
+	return c.String(http.StatusOK, "User-agent: *\nDisallow: /\n")
+}
+
 func (s *server) handleIndex(c echo.Context) error {
 	return c.Blob(http.StatusOK, "text/html; charset=utf-8", indexHTML)
 }