feat: add multi-modal support for images and PDFs with OCR

- Add image attachment support via Discord URLs
- Add PDF/document text extraction using markitdown
- Add OCR fallback for scanned PDFs (ocrmypdf + tesseract)
- Support .env.local for dev environment (falls back to .env for production)
- Extend MessageBuilder to handle text, images, and documents
- Update ClaudeSessionPool to support structured prompts with multi-modal content

Supported formats: PDF, DOCX, XLSX, PPTX + images (JPEG, PNG, GIF, WebP)

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude <noreply@anthropic.com>

Author: jonathanglasmeyer

discord-server/Dockerfile

@@ -37,6 +37,9 @@ RUN apt-get update && apt-get install -y \
     python3 \
     python3-pip \
     wget \
+    ocrmypdf \
+    tesseract-ocr \
+    tesseract-ocr-deu \
     && rm -rf /var/lib/apt/lists/* \
     && apt-get clean
 
@@ -63,6 +66,9 @@ RUN wget -q -O /tmp/claude-install.sh https://claude.ai/install.sh && \
     timeout 120 bash /tmp/claude-install.sh || exit 1 && \
     rm -f /tmp/claude-install.sh
 
+# Install markitdown for PDF/document processing
+RUN /home/node/.local/bin/uv pip install --system markitdown
+
 # Copy application code AFTER CLI install (so code changes don't rebuild Claude CLI)
 COPY . .
 

discord-server/bot.js

@@ -1,5 +1,12 @@
 import dotenv from 'dotenv';
-dotenv.config();
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+
+// Load .env.local first (dev), fallback to .env (production)
+dotenv.config({ path: join(__dirname, '.env.local') });
+dotenv.config(); // Fallback for vars not in .env.local
 
 import express from 'express';
 
@@ -31,6 +38,7 @@ import { ProgressReporter } from './lib/ProgressReporter.js';
 import ThreadNamer from './lib/ThreadNamer.js';
 import { PerformanceTracer } from './lib/PerformanceTracer.js';
 import ClaudeSessionPool from './lib/ClaudeSessionPool.js';
+import { buildStructuredPrompt } from './lib/MessageBuilder.js';
 
 const client = new Client({
   intents: [
@@ -132,7 +140,16 @@ client.on('messageCreate', async (message) => {
 
   if (!isInboxChannel && !isInboxThread) return;
 
+  // Build structured prompt with text, images, and documents
+  const structuredPrompt = await buildStructuredPrompt(message);
+
   console.log(`📝 Processing message from ${message.author.tag}: ${message.content}`);
+  if (structuredPrompt.hasImages) {
+    console.log(`🖼️  Found ${structuredPrompt.images.length} image(s): ${structuredPrompt.images.map(i => i.name).join(', ')}`);
+  }
+  if (structuredPrompt.hasDocuments) {
+    console.log(`📄 Found ${structuredPrompt.documents.length} document(s): ${structuredPrompt.documents.map(d => d.name).join(', ')}`);
+  }
 
   // Initialize performance tracing
   const tracer = new PerformanceTracer();
@@ -168,25 +185,33 @@ client.on('messageCreate', async (message) => {
     await progressReporter.start();
     tracer.endPhase();
 
-    // Build conversation context
+    // Build conversation context - use structured prompt for images and documents
     // Skip conversation context building - sessions handle context
-    const conversationPrompt = message.content;
     tracer.endPhase({
-      promptLength: conversationPrompt.length,
-      promptPreview: conversationPrompt.slice(0, 100) + '...'
+      promptLength: structuredPrompt.text.length,
+      promptPreview: structuredPrompt.text.slice(0, 100) + '...',
+      hasImages: structuredPrompt.hasImages,
+      imageCount: structuredPrompt.images.length,
+      hasDocuments: structuredPrompt.hasDocuments,
+      documentCount: structuredPrompt.documents.length
     });
 
     // Process with Claude Code SDK with retry logic
     console.log('🤖 Calling Claude with conversation context...');
 
     tracer.startPhase('claude_sdk_call', {
-      promptLength: conversationPrompt.length,
+      promptLength: structuredPrompt.text.length,
+      hasImages: structuredPrompt.hasImages,
+      imageCount: structuredPrompt.images.length,
+      hasDocuments: structuredPrompt.hasDocuments,
+      documentCount: structuredPrompt.documents.length,
       vaultPath: process.env.OBSIDIAN_VAULT_PATH || '/srv/claude-jobs/obsidian-vault'
     });
 
     const claudeResult = await errorHandler.retryOperation(async () => {
       // Use session pool instead of direct query
-      const stream = await sessionPool.processMessage(thread.id, conversationPrompt);
+      // Pass structured prompt for image support
+      const stream = await sessionPool.processMessage(thread.id, structuredPrompt);
 
       // Register stream for /stop command
       activeStreams.set(thread.id, {

discord-server/lib/ClaudeSessionPool.js

@@ -1,4 +1,5 @@
 import { query } from '@anthropic-ai/claude-agent-sdk';
+import { createMessageIterable } from './MessageBuilder.js';
 
 class ClaudeSessionPool {
   constructor(redisClient) {
@@ -17,6 +18,12 @@ class ClaudeSessionPool {
 
   async getOrCreateSession(threadId, initialPrompt = null) {
     try {
+      // Normalize prompt - support both string and structured format
+      const isStructured = typeof initialPrompt === 'object' && initialPrompt?.hasImages !== undefined;
+      const textPrompt = isStructured ? initialPrompt.text : initialPrompt;
+      const hasImages = isStructured && initialPrompt.hasImages;
+      const content = isStructured ? initialPrompt.content : null;
+
       // Check memory cache first - if found, use it exclusively
       if (this.sessionCache.has(threadId)) {
         const metadata = this.sessionMetadata.get(threadId);
@@ -40,8 +47,13 @@ class ClaudeSessionPool {
         console.log(`🔄 Resuming session ${persistedSessionId} for thread ${threadId}`);
 
         try {
+          // Use AsyncIterable for messages with images
+          const promptValue = hasImages
+            ? createMessageIterable(content, persistedSessionId)
+            : (textPrompt || "Continue our conversation");
+
           const session = query({
-            prompt: initialPrompt || "Continue our conversation",
+            prompt: promptValue,
             options: {
               resume: persistedSessionId,
               cwd: process.env.OBSIDIAN_VAULT_PATH || '/srv/claude-jobs/obsidian-vault',
@@ -75,8 +87,13 @@ class ClaudeSessionPool {
       // Create new session
       console.log(`🆕 Creating new Claude session for thread ${threadId}`);
 
+      // Use AsyncIterable for messages with images
+      const promptValue = hasImages
+        ? createMessageIterable(content, '')
+        : (textPrompt || "Start new Discord conversation");
+
       const session = query({
-        prompt: initialPrompt || "Start new Discord conversation",
+        prompt: promptValue,
         options: {
           cwd: process.env.OBSIDIAN_VAULT_PATH || '/srv/claude-jobs/obsidian-vault',
           maxTurns: 100,
@@ -111,8 +128,24 @@ class ClaudeSessionPool {
     }
   }
 
+  /**
+   * Process a message with optional image support
+   * @param {string} threadId - Discord thread ID
+   * @param {string|Object} prompt - Either a string or structured prompt from buildStructuredPrompt
+   * @returns {Query} Claude session stream
+   */
   async processMessage(threadId, prompt) {
     try {
+      // Normalize prompt - support both string and structured format
+      const isStructured = typeof prompt === 'object' && prompt.hasImages !== undefined;
+      const textPrompt = isStructured ? prompt.text : prompt;
+      const hasImages = isStructured && prompt.hasImages;
+      const content = isStructured ? prompt.content : null;
+
+      if (hasImages) {
+        console.log(`🖼️  Processing message with ${prompt.images.length} image(s)`);
+      }
+
       // Check if this is a cached session (existing conversation)
       if (this.sessionCache.has(threadId)) {
         console.log(`🔄 Continuing existing session for thread ${threadId}`);
@@ -127,8 +160,14 @@ class ClaudeSessionPool {
 
         // Resume specific session by ID with detailed logging
         console.log(`🔄 Resuming session ${metadata.sessionId} for thread ${threadId}`);
+
+        // Use AsyncIterable for messages with images
+        const promptValue = hasImages
+          ? createMessageIterable(content, metadata.sessionId)
+          : textPrompt;
+
         const continueSession = query({
-          prompt: prompt,
+          prompt: promptValue,
           options: {
             resume: metadata.sessionId,
             cwd: process.env.OBSIDIAN_VAULT_PATH || '/srv/claude-jobs/obsidian-vault',