Fix #2600 change robots.txt to allow a few pages

Author: fnielsen

scholia/app/views.py

@@ -32,6 +32,21 @@
 from ..config import config
 
 
+ROBOTS_TXT = """User-agent: *
+Allow: /$
+Allow: /about
+Allow: /faq
+Allow: /author/$
+Allow: /organization/$
+Allow: /publisher/$
+Allow: /series/$
+Allow: /topic/$
+Allow: /venue/$
+Allow: /work/$
+Disallow: /
+"""
+
+
 class RegexConverter(BaseConverter):
     """Converter for regular expression routes.
 
@@ -2788,8 +2803,6 @@ def show_robots_txt():
     Disallow: /
 
     """
-    ROBOTS_TXT = ('User-agent: *\n'
-                  'Disallow: /\n')
     return Response(ROBOTS_TXT, mimetype="text/plain")