test.sh

#!/bin/bash

# Test script for the log streamer distributor

set -e

echo "🚀 Testing Log Streamer Distributor"
echo "=================================="

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
NC='\033[0m' # No Color

# Function to print colored output
print_status() {
    echo -e "${GREEN}✓${NC} $1"
}

print_warning() {
    echo -e "${YELLOW}⚠${NC} $1"
}

print_error() {
    echo -e "${RED}✗${NC} $1"
}

# Check if Docker is running
if ! docker info > /dev/null 2>&1; then
    print_error "Docker is not running. Please start Docker and try again."
    exit 1
fi

print_status "Docker is running"

# Build and start services
echo ""
echo "📦 Building and starting services..."
docker-compose up --build -d

# Wait for services to be ready
echo ""
echo "🚀 Starting emitter with high throughput of 10000 - this is both a load test and a functionality test"
echo "⏳ Waiting for services to be ready..."
sleep 3

# Test distributor health
echo ""
echo "🏥 Testing distributor health..."
if curl -s http://localhost:8080/health | grep -q "healthy"; then
    print_status "Distributor is healthy"
else
    print_error "Distributor health check failed"
    docker-compose logs distributor
    exit 1
fi

# Test analyzer health
echo ""
echo "🔍 Testing analyzer health..."
for port in 9001 9002 9003; do
    if curl -s http://localhost:$port/health | grep -q "healthy"; then
        print_status "Analyzer on port $port is healthy"
    else
        print_warning "Analyzer on port $port health check failed"
    fi
done

# Test packet ingestion
echo ""
echo "📨 Testing packet ingestion..."
response=$(curl -s -X POST http://localhost:8080/ingest \
  -H "Content-Type: application/json" \
  -d '{
    "emitter_id": "test-emitter",
    "packet_id": "test-packet-1",
    "messages": [
      {
        "timestamp": 1640995200000,
        "level": "INFO",
        "message": "Test log message",
        "metadata": {"service": "test"}
      }
    ]
  }')

if echo "$response" | grep -q "accepted"; then
    print_status "Packet ingestion successful"
else
    print_error "Packet ingestion failed: $response"
fi

# Test metrics endpoint
echo ""
echo "📊 Testing metrics endpoint..."
metrics=$(curl -s http://localhost:8080/metrics-json)
if echo "$metrics" | grep -q "total_packets"; then
    print_status "Metrics endpoint working"
    echo "Current metrics:"
    echo "$metrics" | jq '.' 2>/dev/null || echo "$metrics"
else
    print_warning "Metrics endpoint not responding properly"
fi

echo ""
echo "🔄 Stopping emitter and restarting all other services to reset metrics..."
docker-compose down
docker-compose up -d distributor analyzer-1 analyzer-2 analyzer-3

# Wait for services to be ready after restart
echo "⏳ Waiting for services to be ready after restart..."
sleep 3

# Verify services are healthy after restart
echo "🏥 Verifying services are healthy after restart..."
if curl -s http://localhost:8080/health | grep -q "healthy"; then
    print_status "Distributor is healthy after restart"
else
    print_error "Distributor health check failed after restart"
    docker-compose logs distributor
    exit 1
fi

# Test load generation with mixed packet sizes
echo ""
echo "⚡ Testing load generation with mixed packet sizes..."
echo "Starting two emitters to test message-based routing algorithm..."


# Start first emitter (packet size 5) in background
echo "Starting emitter-1 (packet size 5)..."
timeout 3 docker-compose run --rm -e TARGET_URL=http://distributor:8080/ingest -e RATE=200 -e CONCURRENCY=20 -e PACKET_SIZE=5 -e EMITTER_ID=emitter-1 emitter > /tmp/emitter1.log 2>&1 &
emitter1_pid=$!

# Start second emitter (packet size 20) in background  
echo "Starting emitter-2 (packet size 20)..."
timeout 3 docker-compose run --rm -e TARGET_URL=http://distributor:8080/ingest -e RATE=200 -e CONCURRENCY=20 -e PACKET_SIZE=20 -e EMITTER_ID=emitter-2 emitter > /tmp/emitter2.log 2>&1 &
emitter2_pid=$!

# Wait for emitters to generate enough traffic
echo "Running emitters for 3 seconds to generate mixed traffic..."
sleep 3

# Wait for emitters to finish (they have 3 second timeout)
echo "Waiting for emitters to finish..."
wait $emitter1_pid 2>/dev/null || true
wait $emitter2_pid 2>/dev/null || true

final_metrics=$(curl -s http://localhost:8080/metrics-json)

# Verify message-based weight distribution
echo ""
echo "⚖️  Verifying message-based weight distribution..."
echo "This test uses mixed packet sizes (5 and 20 messages per packet) to verify"
echo "that the algorithm routes by MESSAGE count, not packet count."
echo ""

if command -v jq >/dev/null 2>&1; then
    total_packets=$(echo "$final_metrics" | jq -r '.total_packets // 0')
    total_messages=$(echo "$final_metrics" | jq -r '.total_messages // 0')
    
    if [ "$total_messages" -gt 0 ]; then
        echo "Total packets distributed: $total_packets"
        echo "Total messages distributed: $total_messages"
        echo ""
        echo "Expected MESSAGE distribution (from docker-compose.yml):"
        echo "  analyzer-1: 50% of messages (weight: 0.5)"
        echo "  analyzer-2: 30% of messages (weight: 0.3)" 
        echo "  analyzer-3: 20% of messages (weight: 0.2)"
        echo ""
        echo "Actual MESSAGE distribution:"
        
        for analyzer in analyzer-1 analyzer-2 analyzer-3; do
            packets=$(echo "$final_metrics" | jq -r ".packets_by_analyzer.\"$analyzer\" // 0")
            messages=$(echo "$final_metrics" | jq -r ".messages_by_analyzer.\"$analyzer\" // 0")
            if [ "$total_packets" -gt 0 ]; then
                packet_pct=$(echo "scale=1; $packets * 100 / $total_packets" | bc -l 2>/dev/null || echo "0")
            else
                packet_pct="0"
            fi
            if [ "$total_messages" -gt 0 ]; then
                message_pct=$(echo "scale=1; $messages * 100 / $total_messages" | bc -l 2>/dev/null || echo "0")
            else
                message_pct="0"
            fi
            echo "  $analyzer: $packets packets ($packet_pct%) | $messages messages ($message_pct%)"
        done
        

        if [ "$total_messages" -gt 500 ]; then  # Only check if we have enough messages
            # Calculate average messages per packet for each analyzer
            for analyzer in analyzer-1 analyzer-2 analyzer-3; do
                packets=$(echo "$final_metrics" | jq -r ".packets_by_analyzer.\"$analyzer\" // 0")
                messages=$(echo "$final_metrics" | jq -r ".messages_by_analyzer.\"$analyzer\" // 0")
                if [ "$packets" -gt 0 ]; then
                    avg_msg_per_packet=$(echo "scale=2; $messages / $packets" | bc -l 2>/dev/null || echo "0")
                    echo "  $analyzer: $avg_msg_per_packet messages/packet average"
                fi
            done
            
            # Overall average
            if [ "$total_packets" -gt 0 ]; then
                overall_avg=$(echo "scale=2; $total_messages / $total_packets" | bc -l 2>/dev/null || echo "0")
                echo "  Overall: $overall_avg messages/packet average"
            fi
            
        else
            print_warning "Not enough messages ($total_messages) to verify weight distribution. Need at least 500 messages."
        fi
    else
        print_warning "No messages were distributed during the test"
    fi
else
    print_warning "jq not available - cannot verify weight distribution"
fi

# Test failure scenario
echo ""
echo "💥 Testing failure scenario..."
echo "Stopping analyzer-2..."

# Stop one analyzer
docker-compose stop analyzer-2

# Wait for health checker to detect the failure (health check runs every 2 seconds)
echo "Waiting for health checker to detect analyzer failure..."
sleep 3

# Check health
health_after_failure=$(curl -s http://localhost:8080/health)
echo "Health status after stopping analyzer-2:"
echo "$health_after_failure" | jq '.' 2>/dev/null || echo "$health_after_failure"

if echo "$health_after_failure" | grep -q '"healthy_analyzers":2'; then
    print_status "System correctly detected analyzer failure"
else
    print_warning "System may not have detected analyzer failure"
    echo "Expected: healthy_analyzers: 2, but got:"
    echo "$health_after_failure" | grep -o '"healthy_analyzers":[0-9]*' || echo "Could not parse healthy_analyzers count"
fi

# Test weight redistribution when analyzer is offline
echo ""
echo "🔄 Testing weight redistribution with analyzer offline..."
echo "Starting emitters for 3 seconds to test traffic distribution to remaining analyzers..."

# Get baseline metrics before starting new traffic
baseline_metrics=$(curl -s http://localhost:8080/metrics-json)
baseline_analyzer1=$(echo "$baseline_metrics" | jq -r '.messages_by_analyzer["analyzer-1"] // 0')
baseline_analyzer3=$(echo "$baseline_metrics" | jq -r '.messages_by_analyzer["analyzer-3"] // 0')

# Start emitters in background with proper timeout handling
timeout 3 docker-compose run --rm -e TARGET_URL=http://distributor:8080/ingest -e RATE=300 -e CONCURRENCY=25 -e PACKET_SIZE=8 -e EMITTER_ID=offline-test-1 emitter > /tmp/offline_test1.log 2>&1 &
offline_test1_pid=$!

timeout 3 docker-compose run --rm -e TARGET_URL=http://distributor:8080/ingest -e RATE=300 -e CONCURRENCY=25 -e PACKET_SIZE=12 -e EMITTER_ID=offline-test-2 emitter > /tmp/offline_test2.log 2>&1 &
offline_test2_pid=$!

# Wait for emitters to finish
wait $offline_test1_pid 2>/dev/null || true
wait $offline_test2_pid 2>/dev/null || true

# Get final metrics
final_metrics=$(curl -s http://localhost:8080/metrics-json)
final_analyzer1=$(echo "$final_metrics" | jq -r '.messages_by_analyzer["analyzer-1"] // 0')
final_analyzer3=$(echo "$final_metrics" | jq -r '.messages_by_analyzer["analyzer-3"] // 0')

# Calculate messages received during offline test
messages_analyzer1=$((final_analyzer1 - baseline_analyzer1))
messages_analyzer3=$((final_analyzer3 - baseline_analyzer3))
total_offline_messages=$((messages_analyzer1 + messages_analyzer3))

echo "Messages received during offline test:"
echo "  analyzer-1: $messages_analyzer1 messages"
echo "  analyzer-3: $messages_analyzer3 messages"
echo "  Total: $total_offline_messages messages"

# Verify weight distribution among remaining analyzers
if [ "$total_offline_messages" -gt 100 ]; then
    # Calculate percentages
    if [ "$total_offline_messages" -gt 0 ]; then
        pct_analyzer1=$((messages_analyzer1 * 100 / total_offline_messages))
        pct_analyzer3=$((messages_analyzer3 * 100 / total_offline_messages))
        
        echo ""
        echo "📊 Offline weight distribution analysis:"
        echo "  analyzer-1: $pct_analyzer1% (expected ~71.4% - weight 0.5 out of 0.7 total)"
        echo "  analyzer-3: $pct_analyzer3% (expected ~28.6% - weight 0.2 out of 0.7 total)"
        
        # Expected distribution: analyzer-1 should get 5/7 ≈ 71.4%, analyzer-3 should get 2/7 ≈ 28.6%
        expected_analyzer1=71
        expected_analyzer3=29
        tolerance=10
        
        # Check if distribution matches expected weights
        diff1=$((pct_analyzer1 > expected_analyzer1 ? pct_analyzer1 - expected_analyzer1 : expected_analyzer1 - pct_analyzer1))
        diff3=$((pct_analyzer3 > expected_analyzer3 ? pct_analyzer3 - expected_analyzer3 : expected_analyzer3 - pct_analyzer3))
        
        if [ "$diff1" -le "$tolerance" ] && [ "$diff3" -le "$tolerance" ]; then
            print_status "Weight redistribution is working correctly"
            echo "  ✓ Remaining analyzers are sharing traffic according to their relative weights"
            echo "  ✓ analyzer-1 (50% weight) gets ~71% of traffic"
            echo "  ✓ analyzer-3 (20% weight) gets ~29% of traffic"
        else
            print_warning "Weight redistribution may not be working correctly"
            echo "  ⚠ analyzer-1 difference: $diff1% (expected ≤$tolerance%)"
            echo "  ⚠ analyzer-3 difference: $diff3% (expected ≤$tolerance%)"
        fi
    else
        print_warning "No messages were distributed during offline test"
    fi
else
    print_warning "Not enough messages ($total_offline_messages) to verify offline weight distribution"
fi

# Restart analyzer
echo ""
echo "Restarting analyzer-2..."
docker-compose start analyzer-2

# Wait for health checker to detect recovery (health check runs every 5 seconds)
echo "Waiting for health checker to detect analyzer recovery..."
sleep 8

# Check health again
health_after_recovery=$(curl -s http://localhost:8080/health)
echo "Health status after restarting analyzer-2:"
echo "$health_after_recovery" | jq '.' 2>/dev/null || echo "$health_after_recovery"

if echo "$health_after_recovery" | grep -q '"healthy_analyzers":3'; then
    print_status "System correctly recovered analyzer"
else
    print_warning "System may not have recovered analyzer"
    echo "Expected: healthy_analyzers: 3, but got:"
    echo "$health_after_recovery" | grep -o '"healthy_analyzers":[0-9]*' || echo "Could not parse healthy_analyzers count"
fi

# Cleanup
echo ""
echo "🧹 Cleaning up..."
docker-compose down

echo ""
echo "🎉 All tests completed!"
echo ""
echo "📋 Test Summary:"
echo "  ✓ Message-based routing algorithm verified"
echo "  ✓ Mixed packet sizes (5 and 20 messages) tested"
echo "  ✓ Weight distribution by MESSAGE count validated"
echo "  ✓ Packet atomicity preserved"
echo "  ✓ Failure and recovery scenarios tested"
echo "  ✓ Weight redistribution when analyzer goes offline tested"
echo ""
echo "To run the system manually:"
echo "  docker-compose up --build -d"
echo ""
echo "To test with custom load:"
echo "  # Single emitter with custom packet size"
echo "  docker-compose run --rm -e PACKET_SIZE=10 -e RATE=500 emitter"
echo ""
echo "  # Multiple emitters with different packet sizes"
echo "  docker-compose run --rm -e PACKET_SIZE=5 -e EMITTER_ID=small emitter &"
echo "  docker-compose run --rm -e PACKET_SIZE=20 -e EMITTER_ID=large emitter &"
echo ""
echo "To monitor metrics:"
echo "  watch -n 1 'curl -s http://localhost:8080/metrics-json | jq'"
echo ""
echo "Key metrics to watch:"
echo "  - total_messages: Total messages distributed"
echo "  - messages_by_analyzer: Message count per analyzer"
echo "  - packets_by_analyzer: Packet count per analyzer (for comparison)"