Merge pull request #179 from thestumonkey/merge-mycelia-memories

Merge mycelia memories

Author: thestumonkey

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "extras/mycelia"]
+	path = extras/mycelia
+	url = https://github.com/mycelia-tech/mycelia

Makefile

@@ -19,7 +19,7 @@ export $(shell sed 's/=.*//' config.env | grep -v '^\s*$$' | grep -v '^\s*\#')
 SCRIPTS_DIR := scripts
 K8S_SCRIPTS_DIR := $(SCRIPTS_DIR)/k8s
 
-.PHONY: help menu setup-k8s setup-infrastructure setup-rbac setup-storage-pvc config config-docker config-k8s config-all clean deploy deploy-docker deploy-k8s deploy-k8s-full deploy-infrastructure deploy-apps check-infrastructure check-apps build-backend up-backend down-backend k8s-status k8s-cleanup k8s-purge audio-manage test-robot test-robot-integration test-robot-unit test-robot-endpoints test-robot-specific test-robot-clean
+.PHONY: help menu setup-k8s setup-infrastructure setup-rbac setup-storage-pvc config config-docker config-k8s config-all clean deploy deploy-docker deploy-k8s deploy-k8s-full deploy-infrastructure deploy-apps check-infrastructure check-apps build-backend up-backend down-backend k8s-status k8s-cleanup k8s-purge audio-manage mycelia-sync-status mycelia-sync-all mycelia-sync-user mycelia-check-orphans mycelia-reassign-orphans test-robot test-robot-integration test-robot-unit test-robot-endpoints test-robot-specific test-robot-clean
 
 # Default target
 .DEFAULT_GOAL := menu
@@ -57,6 +57,13 @@ menu: ## Show interactive menu (default)
 	@echo "  check-apps         🔍 Check application services"
 	@echo "  clean              🧹 Clean up generated files"
 	@echo
+	@echo "🔄 Mycelia Sync:"
+	@echo "  mycelia-sync-status      📊 Show Mycelia OAuth sync status"
+	@echo "  mycelia-sync-all         🔄 Sync all Friend-Lite users to Mycelia"
+	@echo "  mycelia-sync-user        👤 Sync specific user (EMAIL=user@example.com)"
+	@echo "  mycelia-check-orphans    🔍 Find orphaned Mycelia objects"
+	@echo "  mycelia-reassign-orphans ♻️  Reassign orphans (EMAIL=admin@example.com)"
+	@echo
 	@echo "Current configuration:"
 	@echo "  DOMAIN: $(DOMAIN)"
 	@echo "  DEPLOYMENT_MODE: $(DEPLOYMENT_MODE)"
@@ -101,6 +108,13 @@ help: ## Show detailed help for all targets
 	@echo "🎵 AUDIO MANAGEMENT:"
 	@echo "  audio-manage       Interactive audio file management"
 	@echo
+	@echo "🔄 MYCELIA SYNC:"
+	@echo "  mycelia-sync-status Show Mycelia OAuth sync status for all users"
+	@echo "  mycelia-sync-all   Sync all Friend-Lite users to Mycelia OAuth"
+	@echo "  mycelia-sync-user  Sync specific user (EMAIL=user@example.com)"
+	@echo "  mycelia-check-orphans Find Mycelia objects without Friend-Lite owner"
+	@echo "  mycelia-reassign-orphans Reassign orphaned objects (EMAIL=admin@example.com)"
+	@echo
 	@echo "🧪 ROBOT FRAMEWORK TESTING:"
 	@echo "  test-robot         Run all Robot Framework tests"
 	@echo "  test-robot-integration Run integration tests only"
@@ -333,6 +347,42 @@ audio-manage: ## Interactive audio file management
 	@echo "🎵 Starting audio file management..."
 	@$(SCRIPTS_DIR)/manage-audio-files.sh
 
+# ========================================
+# MYCELIA SYNC
+# ========================================
+
+mycelia-sync-status: ## Show Mycelia OAuth sync status for all users
+	@echo "📊 Checking Mycelia OAuth sync status..."
+	@cd backends/advanced && uv run python scripts/sync_friendlite_mycelia.py --status
+
+mycelia-sync-all: ## Sync all Friend-Lite users to Mycelia OAuth
+	@echo "🔄 Syncing all Friend-Lite users to Mycelia OAuth..."
+	@echo "⚠️  This will create OAuth credentials for users without them"
+	@read -p "Continue? (y/N): " confirm && [ "$$confirm" = "y" ] || exit 1
+	@cd backends/advanced && uv run python scripts/sync_friendlite_mycelia.py --sync-all
+
+mycelia-sync-user: ## Sync specific user to Mycelia OAuth (usage: make mycelia-sync-user EMAIL=user@example.com)
+	@echo "👤 Syncing specific user to Mycelia OAuth..."
+	@if [ -z "$(EMAIL)" ]; then \
+		echo "❌ EMAIL parameter is required. Usage: make mycelia-sync-user EMAIL=user@example.com"; \
+		exit 1; \
+	fi
+	@cd backends/advanced && uv run python scripts/sync_friendlite_mycelia.py --email $(EMAIL)
+
+mycelia-check-orphans: ## Find Mycelia objects without Friend-Lite owner
+	@echo "🔍 Checking for orphaned Mycelia objects..."
+	@cd backends/advanced && uv run python scripts/sync_friendlite_mycelia.py --check-orphans
+
+mycelia-reassign-orphans: ## Reassign orphaned objects to user (usage: make mycelia-reassign-orphans EMAIL=admin@example.com)
+	@echo "♻️  Reassigning orphaned Mycelia objects..."
+	@if [ -z "$(EMAIL)" ]; then \
+		echo "❌ EMAIL parameter is required. Usage: make mycelia-reassign-orphans EMAIL=admin@example.com"; \
+		exit 1; \
+	fi
+	@echo "⚠️  This will reassign all orphaned objects to: $(EMAIL)"
+	@read -p "Continue? (y/N): " confirm && [ "$$confirm" = "y" ] || exit 1
+	@cd backends/advanced && uv run python scripts/sync_friendlite_mycelia.py --reassign-orphans --target-email $(EMAIL)
+
 # ========================================
 # TESTING TARGETS
 # ========================================

README-K8S.md

@@ -266,15 +266,20 @@ friend-lite/
 
 1. **Clone Repository**
    ```bash
-   # Clone Friend-Lite repository
-   git clone https://github.com/yourusername/friend-lite.git
+   # Clone Friend-Lite repository with submodules
+   git clone --recursive https://github.com/yourusername/friend-lite.git
    cd friend-lite
-   
+
+   # If you already cloned without --recursive, initialize submodules:
+   # git submodule update --init --recursive
+
    # Verify template files are present
    ls -la skaffold.env.template
    ls -la backends/advanced/.env.template
    ```
 
+   > **Note:** The `--recursive` flag downloads the optional Mycelia submodule (an alternative memory backend with timeline visualization). Most deployments use the default Friend-Lite memory system and don't need Mycelia.
+
 2. **Install Required Tools**
 
    **kubectl** (required for Skaffold and Helm):

backends/advanced/.env.template

@@ -99,8 +99,8 @@ QDRANT_BASE_URL=qdrant
 # MEMORY PROVIDER CONFIGURATION
 # ========================================
 
-# Memory Provider: "friend_lite" (default) or "openmemory_mcp"
-# 
+# Memory Provider: "friend_lite" (default), "openmemory_mcp", or "mycelia"
+#
 # Friend-Lite (default): In-house memory system with full control
 # - Custom LLM-powered extraction with individual fact storage
 # - Smart deduplication and memory updates (ADD/UPDATE/DELETE)
@@ -113,6 +113,13 @@ QDRANT_BASE_URL=qdrant
 # - Web UI at http://localhost:8765
 # - Requires external server setup
 #
+# Mycelia: Full-featured personal memory timeline
+# - Voice, screenshots, and text capture
+# - Timeline UI with waveform playback
+# - Conversation extraction and semantic search
+# - OAuth federation for cross-instance sharing
+# - Requires Mycelia server setup (extras/mycelia)
+#
 # See MEMORY_PROVIDERS.md for detailed comparison
 MEMORY_PROVIDER=friend_lite
 
@@ -128,6 +135,19 @@ MEMORY_PROVIDER=friend_lite
 # OPENMEMORY_USER_ID=openmemory
 # OPENMEMORY_TIMEOUT=30
 
+# ----------------------------------------
+# Mycelia Configuration
+# (Only needed if MEMORY_PROVIDER=mycelia)
+# ----------------------------------------
+# First start Mycelia:
+#   cd extras/mycelia && docker compose up -d redis mongo mongo-search
+#   cd extras/mycelia/backend && deno task dev
+#
+# IMPORTANT: JWT_SECRET in Mycelia backend/.env must match AUTH_SECRET_KEY above
+# MYCELIA_URL=http://host.docker.internal:5173
+# MYCELIA_DB=mycelia  # Database name (use mycelia_test for test environment)
+# MYCELIA_TIMEOUT=30
+
 # ========================================
 # OPTIONAL FEATURES
 # ========================================

backends/advanced/docker-compose-test.yml

@@ -38,6 +38,8 @@ services:
       - MEMORY_PROVIDER=${MEMORY_PROVIDER:-friend_lite}
       - OPENMEMORY_MCP_URL=${OPENMEMORY_MCP_URL:-http://host.docker.internal:8765}
       - OPENMEMORY_USER_ID=${OPENMEMORY_USER_ID:-openmemory}
+      - MYCELIA_URL=http://mycelia-backend-test:5173
+      - MYCELIA_DB=mycelia_test
       # Disable speaker recognition in test environment to prevent segment duplication
       - DISABLE_SPEAKER_RECOGNITION=false
       - SPEAKER_SERVICE_URL=https://localhost:8085
@@ -146,6 +148,8 @@ services:
       - MEMORY_PROVIDER=${MEMORY_PROVIDER:-friend_lite}
       - OPENMEMORY_MCP_URL=${OPENMEMORY_MCP_URL:-http://host.docker.internal:8765}
       - OPENMEMORY_USER_ID=${OPENMEMORY_USER_ID:-openmemory}
+      - MYCELIA_URL=http://mycelia-backend-test:5173
+      - MYCELIA_DB=mycelia_test
       - DISABLE_SPEAKER_RECOGNITION=false
       - SPEAKER_SERVICE_URL=https://localhost:8085
       # Set low inactivity timeout for tests (2 seconds instead of 60)
@@ -163,6 +167,60 @@ services:
         condition: service_started
     restart: unless-stopped
 
+  # Mycelia - AI memory and timeline service (test environment)
+  mycelia-backend-test:
+    build:
+      context: ../../extras/mycelia/backend
+      dockerfile: Dockerfile.simple
+    ports:
+      - "5100:5173"  # Test backend port
+    environment:
+      # Shared JWT secret for Friend-Lite authentication (test key)
+      - JWT_SECRET=test-jwt-signing-key-for-integration-tests
+      - SECRET_KEY=test-jwt-signing-key-for-integration-tests
+      # MongoDB connection (test database)
+      - MONGO_URL=mongodb://mongo-test:27017
+      - MONGO_DB=mycelia_test
+      - DATABASE_NAME=mycelia_test
+      # Redis connection (ioredis uses individual host/port, not URL)
+      - REDIS_HOST=redis-test
+      - REDIS_PORT=6379
+    volumes:
+      - ../../extras/mycelia/backend/app:/app/app  # Mount source for development
+    depends_on:
+      mongo-test:
+        condition: service_healthy
+      redis-test:
+        condition: service_started
+    healthcheck:
+      test: ["CMD", "deno", "eval", "fetch('http://localhost:5173/health').then(r => r.ok ? Deno.exit(0) : Deno.exit(1))"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 5s
+    restart: unless-stopped
+    profiles:
+      - mycelia
+
+  mycelia-frontend-test:
+    build:
+      context: ../../extras/mycelia
+      dockerfile: frontend/Dockerfile.simple
+      args:
+        - VITE_API_URL=http://localhost:5100
+    ports:
+      - "3002:8080"  # Nginx serves on 8080 internally
+    environment:
+      - VITE_API_URL=http://localhost:5100
+    volumes:
+      - ../../extras/mycelia/frontend/src:/app/src  # Mount source for development
+    depends_on:
+      mycelia-backend-test:
+        condition: service_healthy
+    restart: unless-stopped
+    profiles:
+      - mycelia
+
   # caddy:
   #   image: caddy:2-alpine
   #   ports:

backends/advanced/scripts/create_mycelia_api_key.py

@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Create a proper Mycelia API key (not OAuth client) for Friend-Lite user."""
+
+import base64
+import os
+import sys
+import secrets
+import hashlib
+from pymongo import MongoClient
+from bson import ObjectId
+from datetime import datetime
+
+# MongoDB configuration
+MONGO_URL = os.getenv("MONGO_URL", "mongodb://localhost:27018")
+MYCELIA_DB = os.getenv("MYCELIA_DB", os.getenv("DATABASE_NAME", "mycelia_test"))
+
+# User ID from JWT or argument
+USER_ID = os.getenv("USER_ID", "692c7727c7b16bdf58d23cd1")  # test user
+
+
+def hash_api_key_with_salt(api_key: str, salt: bytes) -> str:
+    """Hash API key with salt (matches Mycelia's hashApiKey function)."""
+    # SHA256(salt + apiKey) in base64
+    h = hashlib.sha256()
+    h.update(salt)
+    h.update(api_key.encode('utf-8'))
+    return base64.b64encode(h.digest()).decode('utf-8')  # Use base64 like Mycelia
+
+
+def main():
+    print(f"📊 MongoDB Configuration:")
+    print(f"   URL: {MONGO_URL}")
+    print(f"   Database: {MYCELIA_DB}\n")
+
+    print("🔐 Creating Mycelia API Key\n")
+
+    # Generate API key in Mycelia format: mycelia_{random_base64url}
+    random_part = secrets.token_urlsafe(32)
+    api_key = f"mycelia_{random_part}"
+
+    # Generate salt (32 bytes)
+    salt = secrets.token_bytes(32)
+
+    # Hash the API key with salt
+    hashed_key = hash_api_key_with_salt(api_key, salt)
+
+    # Open prefix (first 16 chars for fast lookup)
+    open_prefix = api_key[:16]
+
+    print(f"✅ Generated API Key:")
+    print(f"   Key: {api_key}")
+    print(f"   Open Prefix: {open_prefix}")
+    print(f"   Owner: {USER_ID}\n")
+
+    # Connect to MongoDB
+    client = MongoClient(MONGO_URL)
+    db = client[MYCELIA_DB]
+    api_keys = db["api_keys"]
+
+    # Check for existing active keys for this user
+    existing = api_keys.find_one({"owner": USER_ID, "isActive": True})
+    if existing:
+        print(f"ℹ️  Existing active API key found: {existing['_id']}")
+        print(f"   Deactivating old key...\n")
+        api_keys.update_one(
+            {"_id": existing["_id"]},
+            {"$set": {"isActive": False}}
+        )
+
+    # Create API key document (matches Mycelia's format)
+    api_key_doc = {
+        "hashedKey": hashed_key,  # Note: hashedKey, not hash!
+        "salt": base64.b64encode(salt).decode('utf-8'),  # Store as base64 like Mycelia
+        "owner": USER_ID,
+        "name": "Friend-Lite Integration",
+        "policies": [
+            {
+                "resource": "**",
+                "action": "*",
+                "effect": "allow"
+            }
+        ],
+        "openPrefix": open_prefix,
+        "createdAt": datetime.now(),
+        "isActive": True,
+    }
+
+    # Insert into database
+    result = api_keys.insert_one(api_key_doc)
+    client_id = str(result.inserted_id)
+
+    print(f"🎉 API Key Created Successfully!")
+    print(f"   Client ID: {client_id}")
+    print(f"   API Key: {api_key}")
+    print(f"\n" + "=" * 70)
+    print("📋 MYCELIA CONFIGURATION (Test Environment)")
+    print("=" * 70)
+    print(f"\n1️⃣  Configure Mycelia Frontend Settings:")
+    print(f"   • Go to: http://localhost:3002/settings")
+    print(f"   • API Endpoint: http://localhost:5100")
+    print(f"   • Client ID: {client_id}")
+    print(f"   • Client Secret: {api_key}")
+    print(f"   • Click 'Save' and then 'Test Token'")
+    print(f"\n✅ This API key uses the proper Mycelia format with salt!")
+    print("=" * 70 + "\n")
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())