test-phase3-rag.sh

#!/bin/bash

# Phase 3 Test Script - RAG Pipeline
echo "🧪 Testing Phase 3: RAG Pipeline (Retrieval Augmented Generation)"
echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo ""

# Colors
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
NC='\033[0m'

echo -e "${BLUE}📚 Phase 3 - RAG (Retrieval Augmented Generation)${NC}"
echo "This test verifies that the system can:"
echo "  1. Find similar past failures"
echo "  2. Retrieve historical context"
echo "  3. Enhance AI prompts with past solutions"
echo "  4. Provide context-aware analysis"
echo ""

echo -e "${BLUE}Step 1: Process First Failure (to create history)${NC}"
echo "Sending a test log with npm error..."

TEST_LOG_1='##[group]Build Application
Run npm install
npm ERR! code ENOENT
npm ERR! syscall open
npm ERR! path /home/runner/work/package.json
npm ERR! errno -2
npm ERR! enoent ENOENT: no such file or directory, open '\''/home/runner/work/package.json'\''
##[endgroup]'

curl -s -X POST http://localhost:3001/api/analyze \
  -H "Content-Type: text/plain" \
  -d "$TEST_LOG_1" > /dev/null

echo "✅ First failure logged"
echo ""
sleep 2

echo -e "${BLUE}Step 2: Process Similar Failure (should find history)${NC}"
echo "Sending a similar test log..."

TEST_LOG_2='##[group]Setup Dependencies
npm install failed
Error: cannot find package.json file
ENOENT: no such file or directory
##[endgroup]'

RESPONSE=$(curl -s -X POST http://localhost:3001/api/analyze \
  -H "Content-Type: text/plain" \
  -d "$TEST_LOG_2")

echo "✅ Second failure processed"
echo ""

echo -e "${BLUE}Step 3: Check Worker Logs${NC}"
echo "Look for these RAG indicators in worker output:"
echo "  ✓ '🔍 RAG: Retrieving historical context...'"
echo "  ✓ '✅ RAG: Found X relevant past failures'"
echo "  ✓ '📚 RAG Context: Found X similar case(s)'"
echo "  ✓ '🎯 RAG Enhanced: Found X similar past case(s)'"
echo "  ✓ '📊 Confidence: XX% - ...'"
echo ""

echo -e "${BLUE}Step 4: Analysis Response${NC}"
echo "$RESPONSE" | python3 -m json.tool 2>/dev/null || echo "$RESPONSE"
echo ""

echo -e "${BLUE}Step 5: Expected Results${NC}"
echo "✅ Second analysis should reference or be influenced by first failure"
echo "✅ Higher confidence score (75-95%) if similar cases found"
echo "✅ Analysis quality should improve over time"
echo ""

echo "━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"
echo -e "${GREEN}✅ Phase 3 Test Complete!${NC}"
echo ""
echo "🎉 All 3 Phases Implemented:"
echo "  Phase 1: ✅ Smart Chunking & Enhanced Error Detection"
echo "  Phase 2: ✅ Embeddings & Vector Search"
echo "  Phase 3: ✅ RAG Pipeline & Context-Aware Analysis"
echo ""
echo "🚀 Your CI/CD Analyzer now has:"
echo "  • Unlimited log size handling"
echo "  • 30+ error pattern detection"
echo "  • Semantic understanding (embeddings)"
echo "  • Historical context retrieval"
echo "  • Self-improving analysis (learns from past failures)"
echo "  • Context-aware AI suggestions"
echo ""
echo "💡 Next: Test with real GitHub webhooks to see RAG in action!"