MCP-Server for consuming and operating on news (#25)

heussd · web-flow · commit bc5e8c395f2e · 2025-11-07T19:10:26.000+10:00
diff --git a/.devcontainer/golang/compose.yml b/.devcontainer/golang/compose.yml
diff --git a/.devcontainer/golang/devcontainer.json b/.devcontainer/golang/devcontainer.json
@@ -1,7 +1,7 @@
 {
-    "name": "ⁿⁿᵃ",
+    "name": "Golang",
     "dockerComposeFile": [
-        "../compose.yml", // First file determines root
+        "../../compose.yml", // First file determines root
         "./compose.yml"
     ],
     "service": "devcontainer",
diff --git a/.devcontainer/mcp-news-desk/Dockerfile b/.devcontainer/mcp-news-desk/Dockerfile
@@ -0,0 +1,7 @@
+FROM python:3.12-slim AS base
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
+
+RUN apt update && apt install -y --no-install-recommends \
+    npm \
+    git \
+    make
diff --git a/.devcontainer/mcp-news-desk/compose.yml b/.devcontainer/mcp-news-desk/compose.yml
@@ -0,0 +1,29 @@
+services:
+    feed-feeder: !reset ""
+    article-feeder: !reset ""
+    raindrop-integration: !reset ""
+    watchtower: !reset ""
+    news-feeder: !reset ""
+    nats-server: !reset ""
+    keyword-matcher: !reset ""
+    nats-indexer: !reset ""
+    loadbalancer: !reset ""
+    search-mcp-server:
+        deploy:
+            replicas: 1
+    devcontainer:
+        build:
+            context: .devcontainer/mcp-news-desk/
+        volumes:
+            # Mounts the project folder to '/workspace'. While this file is in .devcontainer,
+            # mounts are relative to the first file in the list, which is a level up.
+            - .:/workspace:cached
+
+        # [Optional] Required for ptrace-based debuggers like C++, Go, and Rust
+        cap_add:
+            - SYS_PTRACE
+        security_opt:
+            - seccomp:unconfined
+
+        # Overrides default command so things don't shut down after the process ends.
+        command: /bin/sh -c "while sleep 1000; do :; done"
diff --git a/.devcontainer/mcp-news-desk/devcontainer.json b/.devcontainer/mcp-news-desk/devcontainer.json
@@ -0,0 +1,28 @@
+{
+    "name": "MCP News Desk",
+    "dockerComposeFile": [
+        "../../compose.yml", // First file determines root
+        "./compose.yml"
+    ],
+    "service": "devcontainer",
+    "workspaceFolder": "/workspace",
+    "customizations": {
+        "vscode": {
+            "settings": {
+                "dotfiles.repository": "https://github.com/heussd/dotfiles",
+                "dotfiles.targetPath": "~/.dotfiles",
+                "dotfiles.installCommand": ".install.sh",
+                "files.eol": "\n",
+                "editor.formatOnSave": true,
+                "remote.extensionKind": {
+                    "ms-azuretools.vscode-docker": "workspace"
+                }
+            },
+            "extensions": [
+                "usernamehw.errorlens",
+                "RooVeterinaryInc.roo-cline"
+            ]
+        }
+    },
+    "shutdownAction": "stopCompose"
+}
diff --git a/.gitignore b/.gitignore
@@ -11,3 +11,4 @@ urls-primary.txt
 urls-secondary.txt
 prom-storage
 __debug_*
+.env
diff --git a/.roo/mcp.json b/.roo/mcp.json
@@ -2,7 +2,27 @@
     "mcpServers": {
         "news-search": {
             "type": "streamable-http",
-            "url": "http://localhost:8000/mcp"
+            "url": "http://search-mcp-server:8000/mcp",
+            "alwaysAllow": [
+                "search",
+                "retrieve_article_full_text",
+                "news_start_page"
+            ]
+        },
+        "youtube-transcript": {
+            "command": "uvx",
+            "args": [
+                "--from",
+                "git+https://github.com/jkawamoto/mcp-youtube-transcript",
+                "mcp-youtube-transcript"
+            ],
+            "alwaysAllow": [
+                "get_transcript",
+                "get_video_info"
+            ],
+            "disabledTools": [
+                "get_timed_transcript"
+            ]
         }
     }
 }
diff --git a/architecture.drawio.svg b/architecture.drawio.svg
diff --git a/compose.yml b/compose.yml
@@ -67,7 +67,7 @@ services:
             - 'RAINDROP_ACCESS_TOKEN=${RAINDROP_ACCESS_TOKEN}'
     nats-indexer:
         deploy:
-            replicas: 2
+            replicas: 1
         image: ghcr.io/heussd/nats-news-analysis/nats-indexer:latest
         environment:
             - 'NATS_URL=nats-server:4222'
@@ -84,6 +84,7 @@ services:
             - 'AI_SEARCH_ENDPOINT=${AI_SEARCH_ENDPOINT}'
             - 'AI_SEARCH_API_KEY=${AI_SEARCH_API_KEY}'
             - 'AI_SEARCH_API_VERSION=${AI_SEARCH_API_VERSION}'
+            - 'RAINDROP_RSS_FEED=${RAINDROP_RSS_FEED}'
     watchtower:
         image: containrrr/watchtower
         volumes:
diff --git a/index.en.md b/index.en.md
@@ -16,6 +16,15 @@ The system has the following NATS queues:
 1. `news` - News texts and their metadata.
 1. `match-urls` - URLs of positive matching articles.
 
+
+The outcomes of this system are:
+
+1. A Raindrop.io collection with articles matching to keywords.
+1. An Azure AI Search index prepared for hybrid search on news.
+1. A Grafana dashboard with metrics.
+1. An MCP-Server that allows consuming 1 and 2 with an agentic LLM (such as [Roo Code](https://roocode.com)).
+
+
 ## Involved services
 
 All services are orchestrated and scaled with `compose.yml`.
@@ -33,11 +42,11 @@ All services are orchestrated and scaled with `compose.yml`.
 
 -   [docker.io/heussd/fivefilters-full-text-rss](https://hub.docker.com/r/heussd/fivefilters-full-text-rss) - Retrieves full text of web pages.
 -   [docker.io/nats](https://hub.docker.com/_/nats) - Event queue, key-value store and deduplication.
--   [NGINX](https://www.nginx.com/) - Simple load balancer / reverse proxy
--   [Prometheus NATS Exporter](https://github.com/nats-io/prometheus-nats-exporter) - Make NATS metrics available to Prometheus
--   [Prometheus](https://prometheus.io/) - Metrics & monitoring
--   [Grafana Loki](https://grafana.com/oss/loki/) - Log aggregation
--   [Grafana](https://grafana.com/grafana/) - Dashboard for metrics and stats
+-   [NGINX](https://www.nginx.com/) - Simple load balancer / reverse proxy.
+-   [Prometheus NATS Exporter](https://github.com/nats-io/prometheus-nats-exporter) - Make NATS metrics available to Prometheus.
+-   [Prometheus](https://prometheus.io/) - Metrics & monitoring.
+-   [Grafana Loki](https://grafana.com/oss/loki/) - Log aggregation.
+-   [Grafana](https://grafana.com/grafana/) - Dashboard for metrics and stats.
 -   [Watchtower](https://github.com/containrrr/watchtower) - Keep containers updated.
 
 ## Message queue for scaling
diff --git a/python/Dockerfile b/python/Dockerfile
@@ -25,7 +25,7 @@ ENV HF_HUB_OFFLINE=1 \
     ORT_DISABLE_ALL_LOGS=1
 
 
-FROM base as devcontainer
+FROM base AS devcontainer
 
 RUN apt update && apt install -y --no-install-recommends \
     golang \
@@ -38,7 +38,7 @@ RUN go install github.com/nats-io/natscli/nats@latest
 ENV PATH="/root/go/bin:${PATH}"
 
 
-FROM base as build
+FROM base AS build
 
 COPY *.py ./
 
@@ -51,12 +51,12 @@ ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1
 
 
-FROM build as search-mcp-server
+FROM build AS search-mcp-server
 
-CMD ["uv", "run", "search-mcp-server.py"]
+CMD ["uv", "run", "mcp-server.py"]
 
 
-FROM build as nats-indexer
+FROM build AS nats-indexer
 
 CMD ["uv", "run", "nats-indexer.py"]
 
diff --git a/python/compose.yml b/python/compose.yml
@@ -1,5 +1,6 @@
 services:
   service:
+    image: ghcr.io/heussd/nats-news-analysis/search-mcp-server:latest
     build:
       context: .
       target: search-mcp-server
diff --git a/python/mcp-server.py b/python/mcp-server.py
@@ -0,0 +1,65 @@
+import json
+import os
+
+from fastmcp import FastMCP
+import requests
+
+import ai_search
+
+mcp = FastMCP("News Desk")
+
+
+@mcp.tool()
+async def news_start_page() -> str:
+    """
+        Shows a news start page as RSS feed. A nice entry point for all kinds of news interests.
+        Use `retrieve_article_full_text` to retrieve the full text for articles.
+        Use `search` to find related news articles based on keywords or topics.
+
+    Args:
+        None
+    """
+    url = os.getenv("RAINDROP_RSS_FEED", "https://www.abc.net.au/news/feed/2942460/rss.xml")
+    spoiler = requests.get(url, verify=False)
+    spoiler.raise_for_status()
+    data = spoiler.text
+    return data
+
+
+@mcp.tool
+def search(query: str) -> str:
+    """
+    Find related news to a query with a multi-language, semantic search. Do not search for generic terms such as "latest technology trends 2025", but search for specific technologies, keywords, names, events, etc.
+
+    Args:
+        query (str): The search query. Can be in any language. Search will be keyword- and embedding-based.
+    """
+    return json.dumps(ai_search.search(query), indent=0)
+
+
+@mcp.tool()
+async def retrieve_article_full_text(url: str) -> any:
+    """
+        Retrieves the full text content and additional metadata of an article from the internet.
+
+    Args:
+        URL: The URL of the article to retrieve the full text of.
+    """
+    try:
+        response = requests.get(
+            f"http://fullfeedrss:80/extract.php", params={"url": url}
+        )
+        response.raise_for_status()
+        data = response.json()
+        return data
+    except Exception as e:
+        from readabilipy import simple_json_from_html_string
+
+        req = requests.get(url)
+        article = simple_json_from_html_string(req.text, use_readability=True)
+
+        return article
+
+
+if __name__ == "__main__":
+    mcp.run(transport="streamable-http", host="0.0.0.0")
diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -11,5 +11,6 @@ dependencies = [
     "nats-py>=2.11.0",
     "numpyencoder>=0.3.2",
     "pydantic>=2.12.3",
+    "readabilipy>=0.3.0",
     "requests>=2.32.5",
 ]
diff --git a/python/search-mcp-server.py b/python/search-mcp-server.py
diff --git a/python/uv.lock b/python/uv.lock
