Create robots.txt file for the website

Signed-off-by: Jiri Podivin <jpodivin@redhat.com>

Author: jpodivin

backend/src/api.py

@@ -18,6 +18,7 @@
     JSONResponse,
     FileResponse,
     RedirectResponse,
+    PlainTextResponse,
 )
 from fastapi.staticfiles import StaticFiles
 from fastapi.templating import Jinja2Templates
@@ -62,6 +63,7 @@
     write_json_file,
     fetch_text,
     sanitize_uploaded_schema,
+    get_robots,
 )
 from src.store import Storator3000
 from src.exceptions import NoDataFound
@@ -634,3 +636,9 @@ def get_report_stats() -> dict:
     LOGGER.info("Retrieving annotation statistics")
 
     return Storator3000.get_stats()
+
+
+@app.get("/robots.txt", include_in_schema=False, response_class=PlainTextResponse)
+def robots() -> str:
+    """Return robots.txt"""
+    return get_robots()

backend/src/constants.py

@@ -47,3 +47,10 @@ class BuildIdTitleEnum(StrEnum):
 LOGGER_NAME = "logdetective_website"
 
 STATIC_SOURCE_DIR = Path(__file__).parent.parent.parent / "frontend" / "public"
+
+DEFAULT_ROBOTS = """
+User-Agent: *
+DisallowAITraining: /
+Content-Usage: ai=n
+Allow: /
+"""

backend/src/spells.py

@@ -9,6 +9,7 @@
 import tarfile
 import tempfile
 from contextlib import contextmanager
+from functools import lru_cache
 from pathlib import Path
 from typing import Any, Iterator, Optional
 
@@ -30,6 +31,7 @@
     html_careful_unescape,
     log_schema_redaction,
 )
+from src.constants import DEFAULT_ROBOTS, STATIC_SOURCE_DIR
 
 
 @contextmanager
@@ -269,3 +271,16 @@ def sanitize_uploaded_schema(input_schema: FeedbackSchema) -> FeedbackSchema:
         )
 
     return result
+
+
+@lru_cache(maxsize=1)
+def get_robots() -> str:
+    """Return contents robots.txt, if the file doesn't exist return built-in default."""
+
+    try:
+        with open(
+            os.path.join(STATIC_SOURCE_DIR, "robots.txt"), encoding="utf-8"
+        ) as robots:
+            return robots.read()
+    except FileNotFoundError:
+        return DEFAULT_ROBOTS

backend/tests/conftest.py

@@ -217,3 +217,12 @@ def container_feedback_input_output_schema_tuple():
             },
         ),
     )
+
+
+@pytest.fixture
+def clear_robots_cache():
+    """Clears cache of get_robots function"""
+    from src.spells import get_robots  # pylint: disable=import-outside-toplevel
+
+    get_robots.cache_clear()
+    yield

backend/tests/unit/test_spells.py

@@ -1,11 +1,13 @@
+from unittest.mock import patch
 import responses
-
+from src.constants import DEFAULT_ROBOTS
 from src.spells import (
     ensure_text,
     fetch_text,
     read_json_file,
     read_text_file,
     write_json_file,
+    get_robots,
 )
 
 
@@ -88,3 +90,20 @@ def test_reads_utf8_content(self, tmp_path):
         result = read_text_file(file_path)
 
         assert result == content
+
+
+class TestGetRobots:
+    """Tests for robots.txt retrieval function"""
+
+    def test_file_retrieval(self, clear_robots_cache):  # pylint: disable=unused-argument
+        """Test that robots.txt file is properly returned"""
+        robots_txt = get_robots()
+        assert isinstance(robots_txt, str)
+        assert robots_txt != DEFAULT_ROBOTS
+
+    @patch("src.spells.STATIC_SOURCE_DIR", "this/path/does/not/exist")
+    def test_default_fallback(self, clear_robots_cache):  # pylint: disable=unused-argument
+        """Test that default fallback is used if file is not accessible"""
+        robots_txt = get_robots()
+        assert isinstance(robots_txt, str)
+        assert robots_txt == DEFAULT_ROBOTS

frontend/public/robots.txt

@@ -0,0 +1,67 @@
+# Well known AI agents and bots are allowed to access the website
+# with exception of the /explain, /contribute and /review paths
+
+User-Agent: GPTBot
+User-Agent: ClaudeBot
+User-Agent: Claude-User
+User-Agent: Claude-SearchBot
+User-Agent: CCBot
+User-Agent: Google-Extended
+User-Agent: Applebot-Extended
+User-Agent: Facebookbot
+User-Agent: Meta-ExternalAgent
+User-Agent: Meta-ExternalFetcher
+User-Agent: diffbot
+User-Agent: PerplexityBot
+User-Agent: Perplexity‑User
+User-Agent: Omgili
+User-Agent: Omgilibot
+User-Agent: webzio-extended
+User-Agent: ImagesiftBot
+User-Agent: Bytespider
+User-Agent: TikTokSpider
+User-Agent: Amazonbot
+User-Agent: Youbot
+User-Agent: SemrushBot-OCOB
+User-Agent: Petalbot
+User-Agent: VelenPublicWebCrawler
+User-Agent: TurnitinBot
+User-Agent: Timpibot
+User-Agent: OAI-SearchBot
+User-Agent: ICC-Crawler
+User-Agent: AI2Bot
+User-Agent: AI2Bot-Dolma
+User-Agent: DataForSeoBot
+User-Agent: AwarioBot
+User-Agent: AwarioSmartBot
+User-Agent: AwarioRssBot
+User-Agent: Google-CloudVertexBot
+User-Agent: PanguBot
+User-Agent: Kangaroo Bot
+User-Agent: Sentibot
+User-Agent: img2dataset
+User-Agent: Meltwater
+User-Agent: Seekr
+User-Agent: peer39_crawler
+User-Agent: cohere-ai
+User-Agent: cohere-training-data-crawler
+User-Agent: DuckAssistBot
+User-Agent: Scrapy
+User-Agent: Cotoyogi
+User-Agent: aiHitBot
+User-Agent: Factset_spyderbot
+User-Agent: FirecrawlAgent
+
+Disallow: /review
+Disallow: /contribute
+Disallow: /explain
+DisallowAITraining: /review
+DisallowAITraining: /contribute
+DisallowAITraining: /explain
+
+# All other bots are blocked from training on the website
+# but allowed to access it for general use
+User-Agent: *
+DisallowAITraining: /
+Content-Usage: ai=n
+Allow: /