feat: Content Engine v2.0 - FastAPI Backend, Distributed n8n Queue Mode, and CI/CD

.github/workflows/content-engine.yml

@@ -0,0 +1,58 @@
+name: Content Engine CI/CD
+
+on:
+  push:
+    branches: [ "main" ]
+    paths:
+      - 'content-engine/**'
+  pull_request:
+    branches: [ "main" ]
+    paths:
+      - 'content-engine/**'
+
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}-content-engine-api
+
+jobs:
+  build-and-test:
+    runs-on: ubuntu-latest
+
+    defaults:
+      run:
+        working-directory: ./content-engine
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log in to the Container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract metadata (tags, labels) for Docker
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+
+      - name: Build and Push FastAPI Image
+        uses: docker/build-push-action@v5
+        with:
+          context: ./content-engine/api
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      # Add testing steps here once unit tests are added to the API
+      # - name: Run tests
+      #   run: |
+      #     docker run --rm ${{ steps.meta.outputs.tags }} pytest

content-engine/.rules

@@ -0,0 +1,7 @@
+# Antigravity & Financial Accuracy Rules
+# Enforces guidelines for floating point precision and physics-engine simulation.
+
+1.  **Exact Representation:** For all budget, token cost, or financial tracking metrics, ALWAYS use the `decimal` module (`from decimal import Decimal`) or `fractions`. Never use standard floating-point numbers (`float`).
+2.  **Comparisons:** Never use the `==` operator for floating point or approximate calculations. Always use `math.isclose()` to verify if two numeric values are effectively equal.
+3.  **Lossless Summation:** When accumulating values (e.g., token costs over a loop), use `math.fsum()` instead of `sum()` to prevent loss of precision.
+4.  **Brax Integration:** For advanced video kinematics or physics-based simulations, utilize `brax` (JAX-based) for hardware-accelerated, parallel environments.

content-engine/api/Dockerfile

@@ -0,0 +1,30 @@
+FROM python:3.14-slim
+
+# Set environment variables to non-interactive and ensure output is unbuffered
+ENV PYTHONUNBUFFERED=1 \
+    PYTHONDONTWRITEBYTECODE=1
+
+WORKDIR /app
+
+# Install system dependencies: FFmpeg and dependencies for Playwright
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    ffmpeg \
+    wget \
+    gnupg \
+    && rm -rf /var/lib/apt/lists/*
+
+# Install python dependencies
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+# Install Playwright browsers (needed for Google Vids automation / Remotion)
+RUN pip install playwright && playwright install --with-deps chromium
+
+# Copy application code
+COPY . .
+
+# Expose the API port
+EXPOSE 8000
+
+# Start the FastAPI application
+CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"]

content-engine/api/main.py

@@ -0,0 +1,84 @@
+from fastapi import FastAPI, HTTPException, Request
+from pydantic import BaseModel
+from decimal import Decimal
+import math
+
+app = FastAPI(
+    title="Automated Social Media Content Engine API",
+    description="FastAPI layer for AI Agents, Video Generation, and NotebookLM ingestion.",
+    version="2.0.0"
+)
+
+# ---------------------------------------------------------
+# Pydantic Schemas
+# ---------------------------------------------------------
+class GenerationRequest(BaseModel):
+    prompt: str
+    target_platform: str
+
+class WebhookPayload(BaseModel):
+    source: str
+    data: dict
+
+# ---------------------------------------------------------
+# RAG and NotebookLM Endpoints
+# ---------------------------------------------------------
+@app.post("/api/v1/notebooklm/ingest", tags=["Data Ingestion"])
+async def ingest_notebooklm_data(payload: WebhookPayload):
+    """
+    Listener for NotebookLM data ingestion from Google Drive.
+    Expects webhooks triggering updates when new source materials are added.
+    """
+    # TODO: Implement webhook validation, parsing, and triggering the agentic workflow.
+    return {"status": "success", "message": "Data received and queued for processing.", "source": payload.source}
+
+@app.post("/api/v1/rag/query", tags=["RAG"])
+async def query_knowledge_base(query: str):
+    """
+    Query the vector store (FAISS/Pinecone) and return grounded answers.
+    """
+    # TODO: Implement integration with Gemini 1.5 Pro 2M context window.
+    return {"status": "success", "answer": f"Simulated answer for: {query}"}
+
+# ---------------------------------------------------------
+# Video Generation Endpoints (Google Vids / Remotion / FFmpeg)
+# ---------------------------------------------------------
+@app.post("/api/v1/video/generate", tags=["Video Generation"])
+async def generate_video(request: GenerationRequest):
+    """
+    Trigger the programmatic video generation pipeline (Remotion/Editly).
+    """
+    # TODO: Orchestrate Director Agent, Producer Agent, and ShortsAgent.
+    return {"status": "processing", "message": "Video generation pipeline triggered.", "job_id": "job_12345"}
+
+@app.post("/api/v1/video/vids/placeholder", tags=["Google Vids"])
+async def trigger_google_vids():
+    """
+    Placeholder endpoint for Google Vids integration.
+    """
+    # TODO: Implement future Google Vids API calls.
+    return {"status": "pending", "message": "Google Vids integration placeholder."}
+
+# ---------------------------------------------------------
+# Utility / Health Check
+# ---------------------------------------------------------
+@app.get("/health", tags=["Health"])
+async def health_check():
+    """
+    Health check endpoint for the Docker container.
+    """
+    # Example of floating point constraint from .rules
+    budget = Decimal("1000.00")
+    cost_per_video = Decimal("10.50")
+    total_cost = cost_per_video * 12
+    remaining_budget = budget - total_cost
+
+    return {
+        "status": "healthy",
+        "daily_volume_target": 12,
+        "remaining_budget_chf": float(remaining_budget) # Cast to float for JSON serialization
+    }
+
+if __name__ == "__main__":
+    import uvicorn
+    uvicorn.run(app, host="0.0.0.0", port=8000)

content-engine/api/requirements.txt

@@ -0,0 +1,7 @@
+fastapi>=0.110.0
+uvicorn>=0.29.0
+pydantic>=2.7.0
+brax>=0.10.0
+jax>=0.4.26
+jaxlib>=0.4.26
+playwright>=1.42.0

content-engine/docker-compose.yml

@@ -0,0 +1,188 @@
+version: "3.8"
+
+volumes:
+  db_storage:
+  n8n_storage:
+  redis_storage:
+  minio_storage:
+
+networks:
+  content_engine_net:
+
+services:
+  # ---------------------------------------------------------
+  # FastAPI Custom Backend (Video Gen & RAG)
+  # ---------------------------------------------------------
+  fastapi_backend:
+    build:
+      context: ./api
+      dockerfile: Dockerfile
+    container_name: fastapi_backend
+    restart: always
+    ports:
+      - "8000:8000"
+    networks:
+      - content_engine_net
+    environment:
+      - WORKSPACE_DIR=/app
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+
+  # ---------------------------------------------------------
+  # Persistence Layer (Postgres 13+ & Redis)
+  # ---------------------------------------------------------
+  postgres:
+    image: postgres:16-alpine
+    container_name: n8n_postgres
+    restart: always
+    environment:
+      - POSTGRES_USER=n8n
+      - POSTGRES_PASSWORD=n8n_secret_password
+      - POSTGRES_DB=n8n
+    volumes:
+      - db_storage:/var/lib/postgresql/data
+    networks:
+      - content_engine_net
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready -h localhost -U n8n -d n8n']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  redis:
+    image: redis:7-alpine
+    container_name: n8n_redis
+    restart: always
+    command: redis-server --appendonly yes
+    volumes:
+      - redis_storage:/data
+    networks:
+      - content_engine_net
+    healthcheck:
+      test: ['CMD', 'redis-cli', 'ping']
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  # ---------------------------------------------------------
+  # Object Storage (MinIO for S3 compatibility)
+  # ---------------------------------------------------------
+  minio:
+    image: minio/minio
+    container_name: n8n_minio
+    restart: always
+    command: server /data --console-address ":9001"
+    environment:
+      - MINIO_ROOT_USER=minioadmin
+      - MINIO_ROOT_PASSWORD=minioadmin123
+    ports:
+      - "9000:9000"
+      - "9001:9001"
+    volumes:
+      - minio_storage:/data
+    networks:
+      - content_engine_net
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 30s
+      timeout: 20s
+      retries: 3
+
+  # ---------------------------------------------------------
+  # n8n Distributed Architecture (Queue Mode)
+  # ---------------------------------------------------------
+  n8n_main:
+    image: docker.n8n.io/n8nio/n8n
+    container_name: n8n_main
+    restart: always
+    ports:
+      - "5678:5678"
+    environment:
+      # Database
+      - DB_TYPE=postgresdb
+      - DB_POSTGRESDB_HOST=postgres
+      - DB_POSTGRESDB_PORT=5432
+      - DB_POSTGRESDB_DATABASE=n8n
+      - DB_POSTGRESDB_USER=n8n
+      - DB_POSTGRESDB_PASSWORD=n8n_secret_password
+      # Queue Mode & Redis
+      - EXECUTIONS_MODE=queue
+      - QUEUE_BULL_REDIS_HOST=redis
+      - QUEUE_BULL_REDIS_PORT=6379
+      - QUEUE_BULL_REDIS_TIMEOUT_THRESHOLD=10000
+      - N8N_GRACEFUL_SHUTDOWN_TIMEOUT=30
+      # Security (Must match across all nodes)
+      - N8N_ENCRYPTION_KEY=super_secret_encryption_key_12345
+      # Disable Webhooks on Main (Routing to webhook processor)
+      - N8N_DISABLE_PRODUCTION_WEBHOOKS_ON_MAIN_PROCESS=true
+      # S3 Storage Configuration
+      - N8N_DEFAULT_BINARY_DATA_MODE=filesystem
+      #- N8N_BINARY_DATA_STORAGE_PATH=/data
+      # Since Queue mode does not support local storage, ideally use S3 variables below,
+      # but standard n8n S3 variables require Enterprise for native binary data offloading.
+      # We will rely on nodes passing data via S3 directly or via shared volume as a fallback if not enterprise.
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    volumes:
+      - n8n_storage:/home/node/.n8n
+    networks:
+      - content_engine_net
+
+  n8n_worker_1:
+    image: docker.n8n.io/n8nio/n8n
+    container_name: n8n_worker_1
+    restart: always
+    command: worker
+    environment:
+      - DB_TYPE=postgresdb
+      - DB_POSTGRESDB_HOST=postgres
+      - DB_POSTGRESDB_PORT=5432
+      - DB_POSTGRESDB_DATABASE=n8n
+      - DB_POSTGRESDB_USER=n8n
+      - DB_POSTGRESDB_PASSWORD=n8n_secret_password
+      - EXECUTIONS_MODE=queue
+      - QUEUE_BULL_REDIS_HOST=redis
+      - QUEUE_BULL_REDIS_PORT=6379
+      - N8N_ENCRYPTION_KEY=super_secret_encryption_key_12345
+      # Worker Specific Concurrency
+      - EXECUTIONS_DATA_MAX_AGE=168
+      - EXECUTIONS_DATA_PRUNE=true
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    networks:
+      - content_engine_net
+
+  n8n_webhook_processor:
+    image: docker.n8n.io/n8nio/n8n
+    container_name: n8n_webhook_processor
+    restart: always
+    command: webhook
+    ports:
+      - "5679:5678"
+    environment:
+      - DB_TYPE=postgresdb
+      - DB_POSTGRESDB_HOST=postgres
+      - DB_POSTGRESDB_PORT=5432
+      - DB_POSTGRESDB_DATABASE=n8n
+      - DB_POSTGRESDB_USER=n8n
+      - DB_POSTGRESDB_PASSWORD=n8n_secret_password
+      - EXECUTIONS_MODE=queue
+      - QUEUE_BULL_REDIS_HOST=redis
+      - QUEUE_BULL_REDIS_PORT=6379
+      - N8N_ENCRYPTION_KEY=super_secret_encryption_key_12345
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+    networks:
+      - content_engine_net