Add news stream, semantic indexer (#24)

* Introduce news stream, semantic indexer

* Update architecture diagram

* Update readme

* Adjust env example

Author: heussd

.devcontainer/compose.yml

@@ -1,19 +1,10 @@
 services:
-    feed-feeder:
-        deploy:
-            replicas: 0
-    article-feeder:
-        deploy:
-            replicas: 0
-    keyword-matcher:
-        deploy:
-            replicas: 0
-    raindrop-integration:
-        deploy:
-            replicas: 0
-    watchtower:
-        deploy:
-            replicas: 0
+    feed-feeder: !reset
+    article-feeder: !reset
+    news-feeder: !reset
+    keyword-matcher: !reset
+    raindrop-integration: !reset
+    watchtower: !reset
     devcontainer:
         build:
             context: ./golang/

.devcontainer/devcontainer.json

@@ -22,7 +22,10 @@
                 "go.testEnvVars": {
                     "CGO_ENABLED": "1"
                 },
-                "go.testFlags": ["-v", "-race"],
+                "go.testFlags": [
+                    "-v",
+                    "-race"
+                ],
                 "go.testTimeout": "10s",
                 "go.coverOnSingleTest": true,
                 "go.coverOnSingleTestFile": true,
@@ -58,7 +61,8 @@
         }
     },
     "containerEnv": {
-        "GOROOT2": "./golang"
+        "GOROOT2": "./golang",
+        "FULLTEXTRSS_SERVER": "http://fullfeedrss:80"
     },
     "shutdownAction": "stopCompose"
-}
+}

.devcontainer/python/compose.yml

@@ -0,0 +1,25 @@
+services:
+    feed-feeder: !reset ""
+    article-feeder: !reset ""
+    raindrop-integration: !reset ""
+    watchtower: !reset ""
+    devcontainer:
+        build:
+            context: ./python/
+            dockerfile: Dockerfile
+            target: devcontainer
+        environment:
+            - NATS_URL=nats-server
+        volumes:
+            # Mounts the project folder to '/workspace'. While this file is in .devcontainer,
+            # mounts are relative to the first file in the list, which is a level up.
+            - .:/workspace:cached
+
+        # [Optional] Required for ptrace-based debuggers like C++, Go, and Rust
+        cap_add:
+            - SYS_PTRACE
+        security_opt:
+            - seccomp:unconfined
+
+        # Overrides default command so things don't shut down after the process ends.
+        command: /bin/sh -c "while sleep 1000; do :; done"

.devcontainer/python/devcontainer.json

@@ -0,0 +1,43 @@
+{
+    "name": "Python",
+    "dockerComposeFile": [
+        "../../compose.yml", // First file determines root
+        "./compose.yml"
+    ],
+    "service": "devcontainer",
+    "workspaceFolder": "/workspace",
+    "customizations": {
+        "vscode": {
+            "settings": {
+                "dotfiles.repository": "https://github.com/heussd/dotfiles",
+                "dotfiles.targetPath": "~/.dotfiles",
+                "dotfiles.installCommand": ".install.sh",
+                "files.eol": "\n",
+                "editor.formatOnSave": true,
+                "remote.extensionKind": {
+                    "ms-azuretools.vscode-docker": "workspace"
+                }
+            },
+            "extensions": [
+                "davidanson.vscode-markdownlint",
+                "ms-azuretools.vscode-docker",
+                "shardulm94.trailing-spaces",
+                "Gruntfuggly.todo-tree",
+                "stkb.rewrap",
+                "github.vscode-pull-request-github",
+                "bajdzis.vscode-database",
+                "IBM.output-colorizer",
+                "github.copilot",
+                "anweber.vscode-httpyac",
+                "ms-python.debugpy",
+                "ms-python.python",
+                "usernamehw.errorlens",
+                "RooVeterinaryInc.roo-cline"
+            ]
+        }
+    },
+    "containerEnv": {
+        "GOROOT2": "./python"
+    },
+    "shutdownAction": "stopCompose"
+}

.env-example

@@ -1,3 +1,7 @@
+AI_SEARCH_ENDPOINT=
+AI_SEARCH_API_KEY=
+AI_SEARCH_API_VERSION=
+
 RAINDROP_ACCESS_TOKEN=
 RAINDROP_COLLECTION=
-KEYWORDS_FILE_URL=
+KEYWORDS_FILE_URL=

.gitignore

@@ -10,3 +10,4 @@ nats-persistence
 urls-primary.txt
 urls-secondary.txt
 prom-storage
+__debug_*

.roo/mcp.json

@@ -0,0 +1,8 @@
+{
+    "mcpServers": {
+        "news-search": {
+            "type": "streamable-http",
+            "url": "http://localhost:8000/mcp"
+        }
+    }
+}