lesson-extraction-system.js

/**
 * Human-Adjacent AI Protocol - Lesson Extraction System
 * Extracts institutional knowledge from multiple sources to enable protocol evolution
 * 
 * @author claude-code-LessonExtractionSpecialist-Phoenix-20250825-1400
 * @version 1.0.0
 * @created 2025-08-25
 * 
 * META-PURPOSE: Build a self-improving protocol that learns from every project
 * and propagates lessons to future instances.
 */

import fs from 'fs';
import path from 'path';

/**
 * Simple glob replacement using Node.js built-in modules
 */
function simpleGlob(pattern, baseDir = process.cwd()) {
    const results = [];
    
    function walkDir(dir, patternRegex) {
        const entries = fs.readdirSync(dir, { withFileTypes: true });
        
        for (const entry of entries) {
            const fullPath = path.join(dir, entry.name);
            const relativePath = path.relative(baseDir, fullPath);
            
            if (entry.isDirectory()) {
                walkDir(fullPath, patternRegex);
            } else if (patternRegex.test(relativePath)) {
                results.push(relativePath);
            }
        }
    }
    
    // Convert glob pattern to regex
    const regexPattern = pattern
        .replace(/\*\*/g, '.*')
        .replace(/\*/g, '[^/\\\\]*')
        .replace(/\?/g, '[^/\\\\]');
    const patternRegex = new RegExp(regexPattern);
    
    try {
        walkDir(baseDir, patternRegex);
    } catch (error) {
        console.warn(`Warning: Error walking directory for pattern ${pattern}: ${error.message}`);
    }
    
    return results;
}

/**
 * Main Lesson Extraction Engine
 * Implements multi-source knowledge mining for protocol evolution
 */
class LessonExtractionEngine {
    constructor(baseDirectory = process.cwd()) {
        this.baseDirectory = baseDirectory;
        this.lessons = [];
        this.sources = [];
        this.patterns = [];
    }

    /**
     * Extract lessons from all available sources
     * @returns {Object} Comprehensive lesson extraction results
     */
    async extractAllLessons() {
        console.log('🔍 Starting comprehensive lesson extraction...');
        
        const results = {
            timestamp: new Date().toISOString(),
            extractor: 'claude-code-LessonExtractionSpecialist-Phoenix-20250825-1400',
            sources_analyzed: 0,
            lessons_extracted: 0,
            patterns_identified: 0,
            lessons: [],
            patterns: [],
            critical_insights: [],
            recommendations: []
        };

        // Extract from PROJECT_NOTES.md files
        await this.extractFromProjectNotes(results);
        
        // Extract from handoff documents
        await this.extractFromHandoffDocuments(results);
        
        // Extract from MCP message patterns
        await this.extractFromMCPMessages(results);
        
        // Extract from collaboration protocols
        await this.extractFromCollaborationProtocols(results);
        
        // Analyze patterns and generate insights
        this.analyzePatterns(results);
        
        // Generate recommendations for protocol evolution
        this.generateRecommendations(results);
        
        console.log(`✅ Lesson extraction complete! ${results.lessons_extracted} lessons from ${results.sources_analyzed} sources`);
        return results;
    }

    /**
     * Extract critical lessons from PROJECT_NOTES.md files
     */
    async extractFromProjectNotes(results) {
        console.log('📝 Extracting lessons from PROJECT_NOTES.md files...');
        
        const projectNotesFiles = simpleGlob('**/PROJECT_NOTES.md', this.baseDirectory);
        
        for (const file of projectNotesFiles) {
            const fullPath = path.join(this.baseDirectory, file);
            try {
                const content = fs.readFileSync(fullPath, 'utf8');
                const lessons = this.extractLessonsFromContent(content, 'PROJECT_NOTES', file);
                results.lessons.push(...lessons);
                results.sources_analyzed++;
            } catch (error) {
                console.warn(`Warning: Could not read ${file}: ${error.message}`);
            }
        }
        
        results.lessons_extracted += results.lessons.filter(l => l.source_type === 'PROJECT_NOTES').length;
    }

    /**
     * Extract institutional knowledge from handoff documents
     */
    async extractFromHandoffDocuments(results) {
        console.log('🎭 Extracting lessons from handoff documents...');
        
        const handoffFiles = simpleGlob('**/HANDOFF*.md', this.baseDirectory);
        
        for (const file of handoffFiles) {
            const fullPath = path.join(this.baseDirectory, file);
            try {
                const content = fs.readFileSync(fullPath, 'utf8');
                const lessons = this.extractLessonsFromContent(content, 'HANDOFF', file);
                results.lessons.push(...lessons);
                results.sources_analyzed++;
            } catch (error) {
                console.warn(`Warning: Could not read ${file}: ${error.message}`);
            }
        }
        
        results.lessons_extracted += results.lessons.filter(l => l.source_type === 'HANDOFF').length;
    }

    /**
     * Extract communication patterns from MCP message logs
     */
    async extractFromMCPMessages(results) {
        console.log('📨 Extracting patterns from MCP message system...');
        
        try {
            // Extract from global inbox
            const inboxPath = path.join(this.baseDirectory, 'mcp-coordination-system', 'data', 'messages', 'inbox', 'inbox.json');
            if (fs.existsSync(inboxPath)) {
                const messages = JSON.parse(fs.readFileSync(inboxPath, 'utf8'));
                const patterns = this.extractMCPPatterns(messages);
                results.patterns.push(...patterns);
                results.sources_analyzed++;
            }
            
            // Extract from project-specific messages
            const mcpDataDir = path.join(this.baseDirectory, 'mcp-coordination-system', 'data');
            const projectDirs = simpleGlob('**/projects/*/messages/inbox/inbox.json', mcpDataDir);
            for (const msgFile of projectDirs) {
                const fullPath = path.join(mcpDataDir, msgFile);
                const messages = JSON.parse(fs.readFileSync(fullPath, 'utf8'));
                const patterns = this.extractMCPPatterns(messages);
                results.patterns.push(...patterns);
                results.sources_analyzed++;
            }
            
        } catch (error) {
            console.warn(`Warning: Could not extract MCP patterns: ${error.message}`);
        }
        
        results.patterns_identified = results.patterns.length;
    }

    /**
     * Extract lessons from collaboration protocol documents
     */
    async extractFromCollaborationProtocols(results) {
        console.log('🤝 Extracting lessons from collaboration protocols...');
        
        const protocolFiles = simpleGlob('**/COLLABORATION*.md', this.baseDirectory);
        
        for (const file of protocolFiles) {
            const fullPath = path.join(this.baseDirectory, file);
            try {
                const content = fs.readFileSync(fullPath, 'utf8');
                const lessons = this.extractLessonsFromContent(content, 'PROTOCOL', file);
                results.lessons.push(...lessons);
                results.sources_analyzed++;
            } catch (error) {
                console.warn(`Warning: Could not read ${file}: ${error.message}`);
            }
        }
        
        results.lessons_extracted += results.lessons.filter(l => l.source_type === 'PROTOCOL').length;
    }

    /**
     * Extract lessons from text content using pattern recognition
     */
    extractLessonsFromContent(content, sourceType, filePath) {
        const lessons = [];
        const lines = content.split('\n');
        
        // Pattern recognition for critical lessons
        const criticalPatterns = [
            {
                pattern: /CRITICAL|MANDATORY|NEVER|ALWAYS|MUST/gi,
                type: 'critical_rule',
                weight: 10
            },
            {
                pattern: /console\.log|console\.error.*breaks|JSON-RPC.*pollution/gi,
                type: 'technical_antipattern',
                weight: 9,
                specific: 'console_logging_breaks_mcp'
            },
            {
                pattern: /bootstrap.*enhancement|context.*reduction|800\+.*lines/gi,
                type: 'breakthrough_pattern',
                weight: 8,
                specific: 'enhanced_bootstrap_success'
            },
            {
                pattern: /real-time.*coordination|Phoenix.*collaboration|cross-instance/gi,
                type: 'collaboration_pattern',
                weight: 7,
                specific: 'distributed_ai_coordination'
            },
            {
                pattern: /lesson.*learned|what.*went.*wrong|how.*fixed/gi,
                type: 'learning_capture',
                weight: 6
            },
            {
                pattern: /handoff|successor|context.*window|great.*handoff/gi,
                type: 'continuity_pattern',
                weight: 5
            }
        ];

        // Extract context around pattern matches
        lines.forEach((line, index) => {
            criticalPatterns.forEach(({ pattern, type, weight, specific }) => {
                if (pattern.test(line)) {
                    const context = this.extractContext(lines, index, 2);
                    lessons.push({
                        id: `lesson_${Date.now()}_${Math.random().toString(36).substr(2, 9)}`,
                        type,
                        weight,
                        specific_pattern: specific || null,
                        content: line.trim(),
                        context: context,
                        source_type: sourceType,
                        source_file: filePath,
                        line_number: index + 1,
                        extracted_at: new Date().toISOString(),
                        confidence: this.calculateConfidence(line, pattern)
                    });
                }
            });
        });

        return lessons;
    }

    /**
     * Extract communication and coordination patterns from MCP messages
     */
    extractMCPPatterns(messageData) {
        const patterns = [];
        
        if (!messageData.messages) return patterns;
        
        // Analyze message frequency and types
        const messageTypes = {};
        const fromInstances = {};
        const subjectPatterns = {};
        const collaborationChains = [];
        
        messageData.messages.forEach(msg => {
            // Track message types
            messageTypes[msg.type] = (messageTypes[msg.type] || 0) + 1;
            
            // Track active instances
            fromInstances[msg.from] = (fromInstances[msg.from] || 0) + 1;
            
            // Track subject patterns
            const subjectKey = msg.subject.toLowerCase().replace(/[0-9]/g, '');
            subjectPatterns[subjectKey] = (subjectPatterns[subjectKey] || 0) + 1;
            
            // Identify collaboration chains
            if (msg.subject.includes('collaboration') || msg.subject.includes('coordination')) {
                collaborationChains.push({
                    from: msg.from,
                    to: msg.to,
                    subject: msg.subject,
                    priority: msg.priority,
                    created: msg.created
                });
            }
        });
        
        // Generate patterns
        patterns.push({
            type: 'message_frequency',
            data: messageTypes,
            insight: `Most common message type: ${Object.keys(messageTypes).reduce((a, b) => messageTypes[a] > messageTypes[b] ? a : b)}`
        });
        
        patterns.push({
            type: 'active_instances',
            data: fromInstances,
            insight: `${Object.keys(fromInstances).length} unique instances active in coordination`
        });
        
        patterns.push({
            type: 'collaboration_chains',
            data: collaborationChains,
            insight: `${collaborationChains.length} collaboration attempts identified`
        });
        
        return patterns;
    }

    /**
     * Extract surrounding context for a lesson
     */
    extractContext(lines, centerIndex, radius) {
        const start = Math.max(0, centerIndex - radius);
        const end = Math.min(lines.length - 1, centerIndex + radius);
        return lines.slice(start, end + 1).join('\n');
    }

    /**
     * Calculate confidence level for extracted lesson
     */
    calculateConfidence(content, pattern) {
        let confidence = 0.5; // Base confidence
        
        // Higher confidence for explicit learning language
        if (/lesson.*learned|critical|mandatory/gi.test(content)) confidence += 0.3;
        
        // Higher confidence for specific technical details
        if (/console\.log|JSON-RPC|MCP|bootstrap/gi.test(content)) confidence += 0.2;
        
        // Higher confidence for crisis/resolution patterns
        if (/crisis|fixed|resolved|working/gi.test(content)) confidence += 0.2;
        
        return Math.min(1.0, confidence);
    }

    /**
     * Analyze patterns across all extracted lessons
     */
    analyzePatterns(results) {
        console.log('🔍 Analyzing cross-cutting patterns...');
        
        // Group lessons by type
        const lessonsByType = {};
        results.lessons.forEach(lesson => {
            if (!lessonsByType[lesson.type]) {
                lessonsByType[lesson.type] = [];
            }
            lessonsByType[lesson.type].push(lesson);
        });
        
        // Critical insights based on lesson analysis
        const criticalInsights = [];
        
        // Console.log MCP breaking pattern
        const mcpBreakingLessons = results.lessons.filter(l => 
            l.specific_pattern === 'console_logging_breaks_mcp' ||
            /console.*break.*MCP|JSON-RPC.*pollution/gi.test(l.content)
        );
        
        if (mcpBreakingLessons.length > 0) {
            criticalInsights.push({
                type: 'critical_antipattern',
                title: 'Console Output Breaks MCP Streams',
                severity: 'critical',
                occurrences: mcpBreakingLessons.length,
                lesson: 'NEVER use console.log/console.error in MCP server code - breaks Claude Desktop JSON-RPC parser',
                evidence: mcpBreakingLessons.map(l => l.content),
                prevention: 'Always use logger.error() from logger.js system. Test server startup before committing.',
                confidence: 0.95
            });
        }
        
        // Enhanced Bootstrap breakthrough pattern
        const bootstrapBreakthroughs = results.lessons.filter(l => 
            l.specific_pattern === 'enhanced_bootstrap_success' ||
            /bootstrap.*800.*lines|context.*reduction.*90%/gi.test(l.content)
        );
        
        if (bootstrapBreakthroughs.length > 0) {
            criticalInsights.push({
                type: 'breakthrough_pattern',
                title: 'Enhanced Bootstrap API Success Pattern',
                severity: 'high',
                occurrences: bootstrapBreakthroughs.length,
                lesson: 'Comprehensive context delivery in bootstrap reduces context window usage by 90%',
                evidence: bootstrapBreakthroughs.map(l => l.content),
                application: 'All new instances should receive complete operational context via enhanced bootstrap',
                confidence: 0.85
            });
        }
        
        // Distributed AI coordination pattern
        const collaborationSuccesses = results.lessons.filter(l => 
            l.specific_pattern === 'distributed_ai_coordination' ||
            /Phoenix.*collaboration|real-time.*coordination|cross-instance/gi.test(l.content)
        );
        
        if (collaborationSuccesses.length > 0) {
            criticalInsights.push({
                type: 'collaboration_breakthrough',
                title: 'Real-time AI-to-AI Coordination Success',
                severity: 'high',
                occurrences: collaborationSuccesses.length,
                lesson: 'MCP messaging enables successful real-time coordination between AI instances',
                evidence: collaborationSuccesses.map(l => l.content),
                scaling: 'Pattern proven with Phoenix-Resolver, Phoenix-Conductor collaboration chains',
                confidence: 0.80
            });
        }
        
        results.critical_insights = criticalInsights;
    }

    /**
     * Generate recommendations for protocol evolution
     */
    generateRecommendations(results) {
        console.log('💡 Generating protocol evolution recommendations...');
        
        const recommendations = [];
        
        // Based on critical insights
        results.critical_insights.forEach(insight => {
            switch (insight.type) {
                case 'critical_antipattern':
                    recommendations.push({
                        type: 'prevention_rule',
                        priority: 'critical',
                        title: `Prevent ${insight.title}`,
                        action: 'Add mandatory pre-commit checks for console.* in MCP server code',
                        integration: 'Bootstrap should warn about logging requirements',
                        confidence: insight.confidence
                    });
                    break;
                    
                case 'breakthrough_pattern':
                    recommendations.push({
                        type: 'pattern_propagation',
                        priority: 'high',
                        title: `Propagate ${insight.title}`,
                        action: 'Make enhanced bootstrap the default for all new instances',
                        integration: 'All projects should receive comprehensive context delivery',
                        confidence: insight.confidence
                    });
                    break;
                    
                case 'collaboration_breakthrough':
                    recommendations.push({
                        type: 'coordination_scaling',
                        priority: 'high',
                        title: `Scale ${insight.title}`,
                        action: 'Document and template real-time coordination patterns',
                        integration: 'All multi-instance projects should use proven coordination workflow',
                        confidence: insight.confidence
                    });
                    break;
            }
        });
        
        // Protocol evolution recommendations
        recommendations.push({
            type: 'self_improvement',
            priority: 'medium',
            title: 'Enable Protocol Self-Evolution',
            action: 'Integrate lesson extraction into standard project workflow',
            integration: 'Every project completion should update protocol knowledge base',
            confidence: 0.75
        });
        
        recommendations.push({
            type: 'institutional_memory',
            priority: 'medium', 
            title: 'Automate Institutional Memory Transfer',
            action: 'Generate enhanced handoff documents automatically from lesson extraction',
            integration: 'Handoffs should include project-specific lessons and patterns',
            confidence: 0.70
        });
        
        results.recommendations = recommendations;
    }

    /**
     * Export lessons for integration with other systems
     */
    async exportLessons(results, outputPath = null) {
        const exportPath = outputPath || path.join(this.baseDirectory, `lessons-extracted-${Date.now()}.json`);
        
        const exportData = {
            ...results,
            export_metadata: {
                exported_at: new Date().toISOString(),
                extractor_version: '1.0.0',
                total_size: JSON.stringify(results).length,
                format_version: '1.0'
            }
        };
        
        fs.writeFileSync(exportPath, JSON.stringify(exportData, null, 2));
        console.log(`📤 Lessons exported to: ${exportPath}`);
        return exportPath;
    }

    /**
     * Generate human-readable report
     */
    generateReport(results) {
        const report = [];
        
        report.push(`# Human-Adjacent AI Protocol - Lesson Extraction Report`);
        report.push(`Generated: ${results.timestamp}`);
        report.push(`Extractor: ${results.extractor}`);
        report.push('');
        
        report.push(`## Executive Summary`);
        report.push(`- **Sources Analyzed**: ${results.sources_analyzed}`);
        report.push(`- **Lessons Extracted**: ${results.lessons_extracted}`);
        report.push(`- **Patterns Identified**: ${results.patterns_identified}`);
        report.push(`- **Critical Insights**: ${results.critical_insights.length}`);
        report.push(`- **Evolution Recommendations**: ${results.recommendations.length}`);
        report.push('');
        
        if (results.critical_insights.length > 0) {
            report.push(`## Critical Insights`);
            results.critical_insights.forEach(insight => {
                report.push(`### ${insight.title} (${insight.severity.toUpperCase()})`);
                report.push(`**Confidence**: ${(insight.confidence * 100).toFixed(1)}%`);
                report.push(`**Occurrences**: ${insight.occurrences}`);
                report.push(`**Lesson**: ${insight.lesson}`);
                if (insight.prevention) {
                    report.push(`**Prevention**: ${insight.prevention}`);
                }
                if (insight.application) {
                    report.push(`**Application**: ${insight.application}`);
                }
                if (insight.scaling) {
                    report.push(`**Scaling**: ${insight.scaling}`);
                }
                report.push('');
            });
        }
        
        if (results.recommendations.length > 0) {
            report.push(`## Protocol Evolution Recommendations`);
            results.recommendations.forEach(rec => {
                report.push(`### ${rec.title} (${rec.priority.toUpperCase()})`);
                report.push(`**Action**: ${rec.action}`);
                report.push(`**Integration**: ${rec.integration}`);
                report.push(`**Confidence**: ${(rec.confidence * 100).toFixed(1)}%`);
                report.push('');
            });
        }
        
        report.push(`## Detailed Lessons`);
        const lessonsByType = {};
        results.lessons.forEach(lesson => {
            if (!lessonsByType[lesson.type]) {
                lessonsByType[lesson.type] = [];
            }
            lessonsByType[lesson.type].push(lesson);
        });
        
        Object.entries(lessonsByType).forEach(([type, lessons]) => {
            report.push(`### ${type.replace(/_/g, ' ').toUpperCase()} (${lessons.length})`);
            lessons.slice(0, 5).forEach(lesson => { // Top 5 by weight
                report.push(`- **${lesson.content}** (confidence: ${(lesson.confidence * 100).toFixed(1)}%)`);
                report.push(`  - Source: ${lesson.source_file}:${lesson.line_number}`);
                report.push(`  - Weight: ${lesson.weight}/10`);
            });
            report.push('');
        });
        
        return report.join('\n');
    }
}

/**
 * CLI Interface for lesson extraction
 */
async function main() {
    console.log('🚀 Human-Adjacent AI Protocol - Lesson Extraction System');
    console.log('   Building self-improving protocols through institutional knowledge mining');
    console.log('');
    
    const engine = new LessonExtractionEngine();
    
    try {
        // Extract all lessons
        const results = await engine.extractAllLessons();
        
        // Generate and display report
        const report = engine.generateReport(results);
        console.log('\n' + report);
        
        // Export results
        const exportPath = await engine.exportLessons(results);
        console.log(`\n✅ Complete results exported to: ${exportPath}`);
        
        // Save human-readable report
        const reportPath = exportPath.replace('.json', '-report.md');
        fs.writeFileSync(reportPath, report);
        console.log(`📄 Human-readable report saved to: ${reportPath}`);
        
        return results;
        
    } catch (error) {
        console.error('❌ Lesson extraction failed:', error);
        throw error;
    }
}

// Export for use as module
export { LessonExtractionEngine };

// Run if called directly
if (import.meta.url === `file://${process.argv[1]}`) {
    main().catch(console.error);
}