ml_pattern_mining.go

package main

import (
	"crypto/sha256"
	"encoding/hex"
	"fmt"
	"log"
	"math"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"
)

// DynamicRule represents a discovered attack pattern
type DynamicRule struct {
	ID             string    `json:"id"`
	Pattern        string    `json:"pattern"`
	Confidence     float64   `json:"confidence"`
	DetectionCount int       `json:"detection_count"`
	FalsePositives int       `json:"false_positives"`
	AutoGenerated  bool      `json:"auto_generated"`
	CreatedAt      time.Time `json:"created_at"`
	LastSeen       time.Time `json:"last_seen"`
	Examples       []string  `json:"examples"`
	Source         string    `json:"source"` // "ngram", "clustering", "similarity"
	Enabled        bool      `json:"enabled"`
	Promoted       bool      `json:"promoted"` // Promoted to main rules
}

// PatternCluster represents a group of similar attack patterns
type PatternCluster struct {
	ID       string
	Patterns []string
	Centroid string
	Size     int
	Variance float64
}

// NGram represents an n-gram with frequency
type NGram struct {
	Text      string
	Frequency int
	Length    int
	Positions []int // Positions where this n-gram appears
}

// PatternMiner discovers new attack patterns from blocked requests
type PatternMiner struct {
	// Storage for blocked requests
	blockedRequests   []BlockedRequest
	maxRequests       int
	mu                sync.RWMutex
	
	// Dynamic rules management
	dynamicRules      map[string]*DynamicRule
	ruleCounter       int
	
	// Pattern analysis
	ngramAnalyzer     *NGramAnalyzer
	clusterAnalyzer   *ClusterAnalyzer
	regexSynthesizer  *RegexSynthesizer
	
	// Configuration
	config            PatternConfig
	minConfidence     float64
	maxFalsePositives int
	promotionThreshold float64
	analysisInterval  time.Duration
	lastAnalysis      time.Time
	
	// False positive tracking
	falsePositiveDB   map[string]int // Pattern hash -> count
	
	// Statistics
	totalPatternsFound    int64
	patternsPromoted      int64
	patternsDeprecated    int64
}

// BlockedRequest represents a request that was blocked
type BlockedRequest struct {
	Timestamp   time.Time
	ClientIP    string
	Method      string
	URL         string
	UserAgent   string
	Payload     string
	RuleID      string
	Reason      string
	Features    *MLFeatures
}

// NGramAnalyzer analyzes n-grams in blocked requests
type NGramAnalyzer struct {
	minLength    int
	maxLength    int
	minFrequency int
	ngrams       map[string]*NGram
	mu           sync.RWMutex
}

// ClusterAnalyzer clusters similar attack patterns
type ClusterAnalyzer struct {
	clusters     []*PatternCluster
	maxClusters  int
	minSimilarity float64
	mu           sync.RWMutex
}

// RegexSynthesizer creates regex patterns from clustered data
type RegexSynthesizer struct {
	patterns map[string]string // Original pattern -> synthesized regex
	mu       sync.RWMutex
}

// NewPatternMiner creates a new pattern mining system with configuration
func NewPatternMiner(config PatternConfig) *PatternMiner {
	pm := &PatternMiner{
		blockedRequests:    make([]BlockedRequest, 0, config.MaxRequestsStored),
		maxRequests:        config.MaxRequestsStored,
		dynamicRules:       make(map[string]*DynamicRule),
		falsePositiveDB:    make(map[string]int),
		config:            config,
		minConfidence:      config.PatternConfidenceMin,
		maxFalsePositives:  config.MaxFalsePositives,
		promotionThreshold: config.PromotionThreshold,
		analysisInterval:   time.Duration(config.AnalysisIntervalMin) * time.Minute,
		lastAnalysis:       time.Now(),
	}
	
	pm.ngramAnalyzer = &NGramAnalyzer{
		minLength:    config.NGramMinLength,
		maxLength:    config.NGramMaxLength,
		minFrequency: config.NGramMinFrequency,
		ngrams:       make(map[string]*NGram),
	}
	
	pm.clusterAnalyzer = &ClusterAnalyzer{
		maxClusters:   config.MaxClusters,
		minSimilarity: config.MinSimilarity,
		clusters:      make([]*PatternCluster, 0, config.MaxClusters),
	}
	
	pm.regexSynthesizer = &RegexSynthesizer{
		patterns: make(map[string]string),
	}
	
	// Start background analysis
	go pm.backgroundAnalysis()
	
	return pm
}

// AddBlockedRequest adds a blocked request for pattern analysis
func (pm *PatternMiner) AddBlockedRequest(req BlockedRequest) {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	
	// Add to storage
	pm.blockedRequests = append(pm.blockedRequests, req)
	
	// Maintain max size
	if len(pm.blockedRequests) > pm.maxRequests {
		pm.blockedRequests = pm.blockedRequests[pm.config.RemoveOldestCount:] // Remove oldest N
	}
	
	// Immediate analysis for high-frequency patterns
	if len(pm.blockedRequests)%pm.config.QuickAnalysisEvery == 0 {
		go pm.quickAnalysis()
	}
}

// backgroundAnalysis runs periodic comprehensive analysis
func (pm *PatternMiner) backgroundAnalysis() {
	ticker := time.NewTicker(pm.analysisInterval)
	defer ticker.Stop()
	
	for range ticker.C {
		pm.performFullAnalysis()
	}
}

// quickAnalysis performs lightweight immediate analysis
func (pm *PatternMiner) quickAnalysis() {
	pm.mu.RLock()
	recentRequests := pm.blockedRequests
	if len(recentRequests) > 100 {
		recentRequests = recentRequests[len(recentRequests)-100:]
	}
	pm.mu.RUnlock()
	
	// Quick n-gram analysis on recent requests
	pm.ngramAnalyzer.analyzeRequests(recentRequests)
	
	// Check for immediate high-confidence patterns
	pm.checkEmergingPatterns()
}

// performFullAnalysis runs comprehensive pattern analysis
func (pm *PatternMiner) performFullAnalysis() {
	pm.mu.RLock()
	requests := make([]BlockedRequest, len(pm.blockedRequests))
	copy(requests, pm.blockedRequests)
	pm.mu.RUnlock()
	
	if len(requests) < 10 {
		return // Need minimum data
	}
	
	log.Printf("ML: Starting full pattern analysis on %d blocked requests", len(requests))
	
	// 1. N-gram analysis
	pm.ngramAnalyzer.analyzeRequests(requests)
	
	// 2. Clustering analysis
	pm.clusterAnalyzer.clusterRequests(requests)
	
	// 3. Pattern synthesis
	pm.synthesizePatterns()
	
	// 4. Validate and promote patterns
	pm.validateAndPromotePatterns()
	
	pm.lastAnalysis = time.Now()
	
	log.Printf("ML: Pattern analysis complete. Found %d total patterns", len(pm.dynamicRules))
}

// analyzeRequests performs n-gram analysis on requests
func (nga *NGramAnalyzer) analyzeRequests(requests []BlockedRequest) {
	nga.mu.Lock()
	defer nga.mu.Unlock()
	
	// Extract payloads for analysis
	var payloads []string
	for _, req := range requests {
		// Combine URL and payload
		combined := req.URL + " " + req.Payload + " " + req.UserAgent
		payloads = append(payloads, strings.ToLower(combined))
	}
	
	// Generate n-grams
	for _, payload := range payloads {
		for length := nga.minLength; length <= nga.maxLength; length++ {
			ngrams := extractNGrams(payload, length)
			for _, ngram := range ngrams {
				if existing, ok := nga.ngrams[ngram]; ok {
					existing.Frequency++
				} else {
					nga.ngrams[ngram] = &NGram{
						Text:      ngram,
						Frequency: 1,
						Length:    length,
						Positions: []int{},
					}
				}
			}
		}
	}
	
	log.Printf("ML: N-gram analysis complete. Found %d unique n-grams", len(nga.ngrams))
}

// extractNGrams extracts n-grams of specified length from text
func extractNGrams(text string, n int) []string {
	if len(text) < n {
		return []string{}
	}
	
	var ngrams []string
	runes := []rune(text)
	
	for i := 0; i <= len(runes)-n; i++ {
		ngram := string(runes[i : i+n])
		// Skip n-grams that are mostly spaces or very common
		if isInterestingNGram(ngram) {
			ngrams = append(ngrams, ngram)
		}
	}
	
	return ngrams
}

// isInterestingNGram filters out uninteresting n-grams
func isInterestingNGram(ngram string) bool {
	// Skip if mostly whitespace
	spaces := strings.Count(ngram, " ")
	if float64(spaces)/float64(len(ngram)) > 0.5 {
		return false
	}
	
	// Skip if too common
	common := []string{"the", "and", "for", "are", "but", "not", "you", "all", "can", "had", "her", "was", "one", "our", "out", "day", "get", "has", "him", "his", "how", "its", "may", "new", "now", "old", "see", "two", "who", "boy", "did", "man", "oil", "sit", "use", "way", "who", "www", "com", "http", "html"}
	for _, word := range common {
		if strings.Contains(ngram, word) {
			return false
		}
	}
	
	// Must contain some special characters or security-relevant content
	hasSpecial := false
	securityChars := []string{"<", ">", "'", "\"", ";", "(", ")", "{", "}", "[", "]", "=", "&", "|", "*", "%", "\\", "/", "?", "#"}
	for _, char := range securityChars {
		if strings.Contains(ngram, char) {
			hasSpecial = true
			break
		}
	}
	
	return hasSpecial || containsSecurityKeywords(ngram)
}

// containsSecurityKeywords checks if n-gram contains security-relevant keywords
func containsSecurityKeywords(text string) bool {
	keywords := []string{
		"select", "union", "insert", "update", "delete", "drop", "exec", "script", "alert", "eval",
		"cmd", "shell", "system", "base64", "decode", "include", "require", "file", "etc", "passwd",
		"admin", "root", "config", "login", "auth", "token", "session", "cookie", "header",
	}
	
	lower := strings.ToLower(text)
	for _, keyword := range keywords {
		if strings.Contains(lower, keyword) {
			return true
		}
	}
	return false
}

// clusterRequests groups similar requests using simple clustering
func (ca *ClusterAnalyzer) clusterRequests(requests []BlockedRequest) {
	ca.mu.Lock()
	defer ca.mu.Unlock()
	
	// Extract unique payloads
	payloadMap := make(map[string][]BlockedRequest)
	for _, req := range requests {
		payload := req.URL + " " + req.Payload
		payloadMap[payload] = append(payloadMap[payload], req)
	}
	
	var payloads []string
	for payload := range payloadMap {
		payloads = append(payloads, payload)
	}
	
	if len(payloads) < 2 {
		return
	}
	
	// Simple clustering based on string similarity
	ca.clusters = ca.clusters[:0] // Reset clusters
	
	for _, payload := range payloads {
		placed := false
		
		// Try to place in existing cluster
		for _, cluster := range ca.clusters {
			similarity := calculateStringSimilarity(payload, cluster.Centroid)
			if similarity >= ca.minSimilarity {
				cluster.Patterns = append(cluster.Patterns, payload)
				cluster.Size++
				// Update centroid (simplified)
				if len(payload) > len(cluster.Centroid) {
					cluster.Centroid = payload
				}
				placed = true
				break
			}
		}
		
		// Create new cluster if not placed
		if !placed && len(ca.clusters) < ca.maxClusters {
			cluster := &PatternCluster{
				ID:       fmt.Sprintf("cluster_%d", len(ca.clusters)),
				Patterns: []string{payload},
				Centroid: payload,
				Size:     1,
				Variance: 0.0,
			}
			ca.clusters = append(ca.clusters, cluster)
		}
	}
	
	log.Printf("ML: Clustering complete. Found %d clusters", len(ca.clusters))
}

// calculateStringSimilarity calculates similarity between two strings
func calculateStringSimilarity(s1, s2 string) float64 {
	if s1 == s2 {
		return 1.0
	}
	
	// Use Jaccard similarity based on character bigrams
	bigrams1 := extractCharBigrams(s1)
	bigrams2 := extractCharBigrams(s2)
	
	intersection := 0
	union := make(map[string]bool)
	
	// Add all bigrams to union
	for bigram := range bigrams1 {
		union[bigram] = true
	}
	for bigram := range bigrams2 {
		union[bigram] = true
	}
	
	// Count intersection
	for bigram := range bigrams1 {
		if bigrams2[bigram] {
			intersection++
		}
	}
	
	if len(union) == 0 {
		return 0.0
	}
	
	return float64(intersection) / float64(len(union))
}

// extractCharBigrams extracts character bigrams from string
func extractCharBigrams(s string) map[string]bool {
	bigrams := make(map[string]bool)
	runes := []rune(s)
	
	for i := 0; i < len(runes)-1; i++ {
		bigram := string(runes[i:i+2])
		bigrams[bigram] = true
	}
	
	return bigrams
}

// synthesizePatterns creates regex patterns from analyzed data
func (pm *PatternMiner) synthesizePatterns() {
	// Synthesize from high-frequency n-grams
	pm.synthesizeFromNGrams()
	
	// Synthesize from clusters
	pm.synthesizeFromClusters()
}

// synthesizeFromNGrams creates patterns from frequent n-grams
func (pm *PatternMiner) synthesizeFromNGrams() {
	pm.ngramAnalyzer.mu.RLock()
	defer pm.ngramAnalyzer.mu.RUnlock()
	
	// Find high-frequency n-grams
	var candidates []*NGram
	for _, ngram := range pm.ngramAnalyzer.ngrams {
		if ngram.Frequency >= pm.ngramAnalyzer.minFrequency {
			candidates = append(candidates, ngram)
		}
	}
	
	// Sort by frequency
	sort.Slice(candidates, func(i, j int) bool {
		return candidates[i].Frequency > candidates[j].Frequency
	})
	
	// Create patterns from top candidates
	for i, ngram := range candidates {
		if i >= pm.config.TopNGramCandidates { // Limit to top N
			break
		}
		
		pattern := pm.createPatternFromNGram(ngram)
		if pattern != "" {
			pm.addDynamicRule(pattern, float64(ngram.Frequency)/100.0, "ngram", []string{ngram.Text})
		}
	}
}

// createPatternFromNGram creates a regex pattern from an n-gram
func (pm *PatternMiner) createPatternFromNGram(ngram *NGram) string {
	text := ngram.Text
	
	// Escape special regex characters
	escaped := regexp.QuoteMeta(text)
	
	// Add some flexibility for common variations
	pattern := escaped
	
	// Replace spaces with flexible whitespace
	pattern = strings.ReplaceAll(pattern, "\\ ", "\\s+")
	
	// Add case insensitive flag
	pattern = "(?i)" + pattern
	
	// Validate the pattern
	if _, err := regexp.Compile(pattern); err != nil {
		return ""
	}
	
	return pattern
}

// synthesizeFromClusters creates patterns from clustered data
func (pm *PatternMiner) synthesizeFromClusters() {
	pm.clusterAnalyzer.mu.RLock()
	defer pm.clusterAnalyzer.mu.RUnlock()
	
	for _, cluster := range pm.clusterAnalyzer.clusters {
		if cluster.Size >= pm.config.MinClusterSize { // Minimum cluster size
			pattern := pm.createPatternFromCluster(cluster)
			if pattern != "" {
				confidence := math.Min(float64(cluster.Size)/10.0, 1.0)
				pm.addDynamicRule(pattern, confidence, "clustering", cluster.Patterns[:min(3, len(cluster.Patterns))])
			}
		}
	}
}

// createPatternFromCluster creates a regex pattern from a cluster
func (pm *PatternMiner) createPatternFromCluster(cluster *PatternCluster) string {
	if len(cluster.Patterns) == 0 {
		return ""
	}
	
	// Find common subsequences
	common := findCommonSubsequence(cluster.Patterns)
	if len(common) < 3 {
		return ""
	}
	
	// Create flexible pattern
	pattern := regexp.QuoteMeta(common)
	pattern = strings.ReplaceAll(pattern, "\\ ", "\\s*")
	pattern = "(?i)" + pattern
	
	// Validate the pattern
	if _, err := regexp.Compile(pattern); err != nil {
		return ""
	}
	
	return pattern
}

// findCommonSubsequence finds the longest common subsequence
func findCommonSubsequence(patterns []string) string {
	if len(patterns) == 0 {
		return ""
	}
	
	reference := patterns[0]
	
	// Find longest common substring
	for length := len(reference); length >= 3; length-- {
		for start := 0; start <= len(reference)-length; start++ {
			substr := reference[start : start+length]
			
			// Check if this substring appears in all patterns
			foundInAll := true
			for _, pattern := range patterns[1:] {
				if !strings.Contains(strings.ToLower(pattern), strings.ToLower(substr)) {
					foundInAll = false
					break
				}
			}
			
			if foundInAll {
				return substr
			}
		}
	}
	
	return ""
}

// addDynamicRule adds a new dynamic rule
func (pm *PatternMiner) addDynamicRule(pattern string, confidence float64, source string, examples []string) {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	
	// Create unique ID
	hasher := sha256.New()
	hasher.Write([]byte(pattern))
	id := "dyn_" + hex.EncodeToString(hasher.Sum(nil))[:8]
	
	// Check if rule already exists
	if existing, ok := pm.dynamicRules[id]; ok {
		existing.DetectionCount++
		existing.LastSeen = time.Now()
		existing.Confidence = math.Max(existing.Confidence, confidence)
		return
	}
	
	// Create new rule
	rule := &DynamicRule{
		ID:             id,
		Pattern:        pattern,
		Confidence:     confidence,
		DetectionCount: 1,
		FalsePositives: 0,
		AutoGenerated:  true,
		CreatedAt:      time.Now(),
		LastSeen:       time.Now(),
		Examples:       examples,
		Source:         source,
		Enabled:        confidence >= pm.minConfidence,
		Promoted:       false,
	}
	
	pm.dynamicRules[id] = rule
	pm.totalPatternsFound++
	
	log.Printf("ML: Created dynamic rule %s (confidence: %.2f, source: %s)", id, confidence, source)
}

// validateAndPromotePatterns validates rules and promotes high-confidence ones
func (pm *PatternMiner) validateAndPromotePatterns() {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	
	for id, rule := range pm.dynamicRules {
		// Calculate adjusted confidence
		adjustedConfidence := rule.Confidence
		
		// Reduce confidence based on false positives
		if rule.FalsePositives > 0 {
			fpRatio := float64(rule.FalsePositives) / float64(rule.DetectionCount+rule.FalsePositives)
			adjustedConfidence *= (1.0 - fpRatio)
		}
		
		// Boost confidence for frequently detected patterns
		if rule.DetectionCount > 10 {
			adjustedConfidence *= 1.1
		}
		
		// Check for promotion
		if !rule.Promoted && adjustedConfidence >= pm.promotionThreshold && rule.FalsePositives <= pm.maxFalsePositives {
			rule.Promoted = true
			pm.patternsPromoted++
			log.Printf("ML: Promoted dynamic rule %s to production (confidence: %.2f)", id, adjustedConfidence)
		}
		
		// Check for deprecation
		if rule.FalsePositives > pm.maxFalsePositives || adjustedConfidence < 0.3 {
			rule.Enabled = false
			pm.patternsDeprecated++
			log.Printf("ML: Deprecated dynamic rule %s (confidence: %.2f, FP: %d)", id, adjustedConfidence, rule.FalsePositives)
		}
	}
}

// ReportFalsePositive reports a false positive for a dynamic rule
func (pm *PatternMiner) ReportFalsePositive(ruleID string) {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	
	if rule, ok := pm.dynamicRules[ruleID]; ok {
		rule.FalsePositives++
		log.Printf("ML: False positive reported for rule %s (total: %d)", ruleID, rule.FalsePositives)
	}
}

// GetDynamicRules returns all dynamic rules
func (pm *PatternMiner) GetDynamicRules() map[string]*DynamicRule {
	pm.mu.RLock()
	defer pm.mu.RUnlock()
	
	// Return a copy
	rules := make(map[string]*DynamicRule)
	for k, v := range pm.dynamicRules {
		ruleCopy := *v
		rules[k] = &ruleCopy
	}
	
	return rules
}

// GetPromotedRules returns rules that should be promoted to main ruleset
func (pm *PatternMiner) GetPromotedRules() []*DynamicRule {
	pm.mu.RLock()
	defer pm.mu.RUnlock()
	
	var promoted []*DynamicRule
	for _, rule := range pm.dynamicRules {
		if rule.Promoted && rule.Enabled {
			ruleCopy := *rule
			promoted = append(promoted, &ruleCopy)
		}
	}
	
	return promoted
}

// GetStats returns pattern mining statistics
func (pm *PatternMiner) GetStats() map[string]interface{} {
	pm.mu.RLock()
	defer pm.mu.RUnlock()
	
	enabledRules := 0
	promotedRules := 0
	
	for _, rule := range pm.dynamicRules {
		if rule.Enabled {
			enabledRules++
		}
		if rule.Promoted {
			promotedRules++
		}
	}
	
	return map[string]interface{}{
		"total_blocked_requests":  len(pm.blockedRequests),
		"total_patterns_found":    pm.totalPatternsFound,
		"patterns_promoted":       pm.patternsPromoted,
		"patterns_deprecated":     pm.patternsDeprecated,
		"active_dynamic_rules":    enabledRules,
		"promoted_rules":          promotedRules,
		"last_analysis":           pm.lastAnalysis,
		"ngram_count":             len(pm.ngramAnalyzer.ngrams),
		"cluster_count":           len(pm.clusterAnalyzer.clusters),
		"false_positive_patterns": len(pm.falsePositiveDB),
	}
}

// checkEmergingPatterns checks for immediate high-confidence patterns
func (pm *PatternMiner) checkEmergingPatterns() {
	// Quick check for very obvious patterns
	pm.ngramAnalyzer.mu.RLock()
	defer pm.ngramAnalyzer.mu.RUnlock()
	
	for _, ngram := range pm.ngramAnalyzer.ngrams {
		if ngram.Frequency >= pm.config.EmergencyFrequency && len(ngram.Text) >= pm.config.MinPatternLength {
			// High frequency + reasonable length = immediate pattern
			pattern := pm.createPatternFromNGram(ngram)
			if pattern != "" {
				pm.addDynamicRule(pattern, pm.config.EmergencyConfidence, "emergency", []string{ngram.Text})
			}
		}
	}
}

// GetPatternStats returns pattern mining statistics (alias for GetStats)
func (pm *PatternMiner) GetPatternStats() map[string]interface{} {
	return pm.GetStats()
}

// PromoteRule manually promotes a rule to permanent status
func (pm *PatternMiner) PromoteRule(pattern string) bool {
	pm.mu.Lock()
	defer pm.mu.Unlock()
	
	// Find rule by pattern
	for _, rule := range pm.dynamicRules {
		if rule.Pattern == pattern {
			rule.Promoted = true
			pm.patternsPromoted++
			log.Printf("ML: Manually promoted rule %s", rule.ID)
			return true
		}
	}
	
	return false
}

// AnalyzeBlockedRequest is a convenience method to add blocked requests
func (pm *PatternMiner) AnalyzeBlockedRequest(payload, ruleID string) {
	req := BlockedRequest{
		Timestamp: time.Now(),
		Payload:   payload,
		RuleID:    ruleID,
		Reason:    "rule_match",
	}
	pm.AddBlockedRequest(req)
}

// Helper function
func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}