diff --git a/.github/workflows/content-engine.yml b/.github/workflows/content-engine.yml new file mode 100644 index 0000000..87af0cb --- /dev/null +++ b/.github/workflows/content-engine.yml @@ -0,0 +1,58 @@ +name: Content Engine CI/CD + +on: + push: + branches: [ "main" ] + paths: + - 'content-engine/**' + pull_request: + branches: [ "main" ] + paths: + - 'content-engine/**' + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }}-content-engine-api + +jobs: + build-and-test: + runs-on: ubuntu-latest + + defaults: + run: + working-directory: ./content-engine + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + + - name: Build and Push FastAPI Image + uses: docker/build-push-action@v5 + with: + context: ./content-engine/api + push: ${{ github.event_name != 'pull_request' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha + cache-to: type=gha,mode=max + + # Add testing steps here once unit tests are added to the API + # - name: Run tests + # run: | + # docker run --rm ${{ steps.meta.outputs.tags }} pytest diff --git a/content-engine/.rules b/content-engine/.rules new file mode 100644 index 0000000..b579591 --- /dev/null +++ b/content-engine/.rules @@ -0,0 +1,7 @@ +# Antigravity & Financial Accuracy Rules +# Enforces guidelines for floating point precision and physics-engine simulation. + +1. **Exact Representation:** For all budget, token cost, or financial tracking metrics, ALWAYS use the `decimal` module (`from decimal import Decimal`) or `fractions`. Never use standard floating-point numbers (`float`). +2. **Comparisons:** Never use the `==` operator for floating point or approximate calculations. Always use `math.isclose()` to verify if two numeric values are effectively equal. +3. **Lossless Summation:** When accumulating values (e.g., token costs over a loop), use `math.fsum()` instead of `sum()` to prevent loss of precision. +4. **Brax Integration:** For advanced video kinematics or physics-based simulations, utilize `brax` (JAX-based) for hardware-accelerated, parallel environments. diff --git a/content-engine/api/Dockerfile b/content-engine/api/Dockerfile new file mode 100644 index 0000000..5609e08 --- /dev/null +++ b/content-engine/api/Dockerfile @@ -0,0 +1,30 @@ +FROM python:3.14-slim + +# Set environment variables to non-interactive and ensure output is unbuffered +ENV PYTHONUNBUFFERED=1 \ + PYTHONDONTWRITEBYTECODE=1 + +WORKDIR /app + +# Install system dependencies: FFmpeg and dependencies for Playwright +RUN apt-get update && apt-get install -y --no-install-recommends \ + ffmpeg \ + wget \ + gnupg \ + && rm -rf /var/lib/apt/lists/* + +# Install python dependencies +COPY requirements.txt . +RUN pip install --no-cache-dir -r requirements.txt + +# Install Playwright browsers (needed for Google Vids automation / Remotion) +RUN pip install playwright && playwright install --with-deps chromium + +# Copy application code +COPY . . + +# Expose the API port +EXPOSE 8000 + +# Start the FastAPI application +CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/content-engine/api/main.py b/content-engine/api/main.py new file mode 100644 index 0000000..9d532c8 --- /dev/null +++ b/content-engine/api/main.py @@ -0,0 +1,84 @@ +from fastapi import FastAPI, HTTPException, Request +from pydantic import BaseModel +from decimal import Decimal +import math + +app = FastAPI( + title="Automated Social Media Content Engine API", + description="FastAPI layer for AI Agents, Video Generation, and NotebookLM ingestion.", + version="2.0.0" +) + +# --------------------------------------------------------- +# Pydantic Schemas +# --------------------------------------------------------- +class GenerationRequest(BaseModel): + prompt: str + target_platform: str + +class WebhookPayload(BaseModel): + source: str + data: dict + +# --------------------------------------------------------- +# RAG and NotebookLM Endpoints +# --------------------------------------------------------- +@app.post("/api/v1/notebooklm/ingest", tags=["Data Ingestion"]) +async def ingest_notebooklm_data(payload: WebhookPayload): + """ + Listener for NotebookLM data ingestion from Google Drive. + Expects webhooks triggering updates when new source materials are added. + """ + # TODO: Implement webhook validation, parsing, and triggering the agentic workflow. + return {"status": "success", "message": "Data received and queued for processing.", "source": payload.source} + +@app.post("/api/v1/rag/query", tags=["RAG"]) +async def query_knowledge_base(query: str): + """ + Query the vector store (FAISS/Pinecone) and return grounded answers. + """ + # TODO: Implement integration with Gemini 1.5 Pro 2M context window. + return {"status": "success", "answer": f"Simulated answer for: {query}"} + +# --------------------------------------------------------- +# Video Generation Endpoints (Google Vids / Remotion / FFmpeg) +# --------------------------------------------------------- +@app.post("/api/v1/video/generate", tags=["Video Generation"]) +async def generate_video(request: GenerationRequest): + """ + Trigger the programmatic video generation pipeline (Remotion/Editly). + """ + # TODO: Orchestrate Director Agent, Producer Agent, and ShortsAgent. + return {"status": "processing", "message": "Video generation pipeline triggered.", "job_id": "job_12345"} + +@app.post("/api/v1/video/vids/placeholder", tags=["Google Vids"]) +async def trigger_google_vids(): + """ + Placeholder endpoint for Google Vids integration. + """ + # TODO: Implement future Google Vids API calls. + return {"status": "pending", "message": "Google Vids integration placeholder."} + +# --------------------------------------------------------- +# Utility / Health Check +# --------------------------------------------------------- +@app.get("/health", tags=["Health"]) +async def health_check(): + """ + Health check endpoint for the Docker container. + """ + # Example of floating point constraint from .rules + budget = Decimal("1000.00") + cost_per_video = Decimal("10.50") + total_cost = cost_per_video * 12 + remaining_budget = budget - total_cost + + return { + "status": "healthy", + "daily_volume_target": 12, + "remaining_budget_chf": float(remaining_budget) # Cast to float for JSON serialization + } + +if __name__ == "__main__": + import uvicorn + uvicorn.run(app, host="0.0.0.0", port=8000) diff --git a/content-engine/api/requirements.txt b/content-engine/api/requirements.txt new file mode 100644 index 0000000..415d4b9 --- /dev/null +++ b/content-engine/api/requirements.txt @@ -0,0 +1,7 @@ +fastapi>=0.110.0 +uvicorn>=0.29.0 +pydantic>=2.7.0 +brax>=0.10.0 +jax>=0.4.26 +jaxlib>=0.4.26 +playwright>=1.42.0 diff --git a/content-engine/docker-compose.yml b/content-engine/docker-compose.yml new file mode 100644 index 0000000..bd0ad21 --- /dev/null +++ b/content-engine/docker-compose.yml @@ -0,0 +1,188 @@ +version: "3.8" + +volumes: + db_storage: + n8n_storage: + redis_storage: + minio_storage: + +networks: + content_engine_net: + +services: + # --------------------------------------------------------- + # FastAPI Custom Backend (Video Gen & RAG) + # --------------------------------------------------------- + fastapi_backend: + build: + context: ./api + dockerfile: Dockerfile + container_name: fastapi_backend + restart: always + ports: + - "8000:8000" + networks: + - content_engine_net + environment: + - WORKSPACE_DIR=/app + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:8000/health"] + interval: 30s + timeout: 10s + retries: 3 + + # --------------------------------------------------------- + # Persistence Layer (Postgres 13+ & Redis) + # --------------------------------------------------------- + postgres: + image: postgres:16-alpine + container_name: n8n_postgres + restart: always + environment: + - POSTGRES_USER=n8n + - POSTGRES_PASSWORD=n8n_secret_password + - POSTGRES_DB=n8n + volumes: + - db_storage:/var/lib/postgresql/data + networks: + - content_engine_net + healthcheck: + test: ['CMD-SHELL', 'pg_isready -h localhost -U n8n -d n8n'] + interval: 10s + timeout: 5s + retries: 5 + + redis: + image: redis:7-alpine + container_name: n8n_redis + restart: always + command: redis-server --appendonly yes + volumes: + - redis_storage:/data + networks: + - content_engine_net + healthcheck: + test: ['CMD', 'redis-cli', 'ping'] + interval: 10s + timeout: 5s + retries: 5 + + # --------------------------------------------------------- + # Object Storage (MinIO for S3 compatibility) + # --------------------------------------------------------- + minio: + image: minio/minio + container_name: n8n_minio + restart: always + command: server /data --console-address ":9001" + environment: + - MINIO_ROOT_USER=minioadmin + - MINIO_ROOT_PASSWORD=minioadmin123 + ports: + - "9000:9000" + - "9001:9001" + volumes: + - minio_storage:/data + networks: + - content_engine_net + healthcheck: + test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] + interval: 30s + timeout: 20s + retries: 3 + + # --------------------------------------------------------- + # n8n Distributed Architecture (Queue Mode) + # --------------------------------------------------------- + n8n_main: + image: docker.n8n.io/n8nio/n8n + container_name: n8n_main + restart: always + ports: + - "5678:5678" + environment: + # Database + - DB_TYPE=postgresdb + - DB_POSTGRESDB_HOST=postgres + - DB_POSTGRESDB_PORT=5432 + - DB_POSTGRESDB_DATABASE=n8n + - DB_POSTGRESDB_USER=n8n + - DB_POSTGRESDB_PASSWORD=n8n_secret_password + # Queue Mode & Redis + - EXECUTIONS_MODE=queue + - QUEUE_BULL_REDIS_HOST=redis + - QUEUE_BULL_REDIS_PORT=6379 + - QUEUE_BULL_REDIS_TIMEOUT_THRESHOLD=10000 + - N8N_GRACEFUL_SHUTDOWN_TIMEOUT=30 + # Security (Must match across all nodes) + - N8N_ENCRYPTION_KEY=super_secret_encryption_key_12345 + # Disable Webhooks on Main (Routing to webhook processor) + - N8N_DISABLE_PRODUCTION_WEBHOOKS_ON_MAIN_PROCESS=true + # S3 Storage Configuration + - N8N_DEFAULT_BINARY_DATA_MODE=filesystem + #- N8N_BINARY_DATA_STORAGE_PATH=/data + # Since Queue mode does not support local storage, ideally use S3 variables below, + # but standard n8n S3 variables require Enterprise for native binary data offloading. + # We will rely on nodes passing data via S3 directly or via shared volume as a fallback if not enterprise. + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + volumes: + - n8n_storage:/home/node/.n8n + networks: + - content_engine_net + + n8n_worker_1: + image: docker.n8n.io/n8nio/n8n + container_name: n8n_worker_1 + restart: always + command: worker + environment: + - DB_TYPE=postgresdb + - DB_POSTGRESDB_HOST=postgres + - DB_POSTGRESDB_PORT=5432 + - DB_POSTGRESDB_DATABASE=n8n + - DB_POSTGRESDB_USER=n8n + - DB_POSTGRESDB_PASSWORD=n8n_secret_password + - EXECUTIONS_MODE=queue + - QUEUE_BULL_REDIS_HOST=redis + - QUEUE_BULL_REDIS_PORT=6379 + - N8N_ENCRYPTION_KEY=super_secret_encryption_key_12345 + # Worker Specific Concurrency + - EXECUTIONS_DATA_MAX_AGE=168 + - EXECUTIONS_DATA_PRUNE=true + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + networks: + - content_engine_net + + n8n_webhook_processor: + image: docker.n8n.io/n8nio/n8n + container_name: n8n_webhook_processor + restart: always + command: webhook + ports: + - "5679:5678" + environment: + - DB_TYPE=postgresdb + - DB_POSTGRESDB_HOST=postgres + - DB_POSTGRESDB_PORT=5432 + - DB_POSTGRESDB_DATABASE=n8n + - DB_POSTGRESDB_USER=n8n + - DB_POSTGRESDB_PASSWORD=n8n_secret_password + - EXECUTIONS_MODE=queue + - QUEUE_BULL_REDIS_HOST=redis + - QUEUE_BULL_REDIS_PORT=6379 + - N8N_ENCRYPTION_KEY=super_secret_encryption_key_12345 + depends_on: + postgres: + condition: service_healthy + redis: + condition: service_healthy + networks: + - content_engine_net