harness.go

package main

import (
	"context"
	"fmt"
	"math/rand"
	"sync"
	"time"
)

// CrashInjectionHarness provides deterministic failure injection
// for testing concurrent workflow system recovery paths.
type CrashInjectionHarness struct {
	mu       sync.Mutex
	Seed     int64
	Failures map[string]int // failure type -> count
}

type FailureType string

const (
	PanicRecovery       FailureType = "panic"
	OOMPath             FailureType = "oom"
	NetworkPartition    FailureType = "network_partition"
	DeadlockDetection   FailureType = "deadlock"
	SchedulerStall      FailureType = "scheduler_stall"
	ResourceExhaustion  FailureType = "resource_exhaustion"
)

func NewCrashInjectionHarness(seed int64) *CrashInjectionHarness {
	return &CrashInjectionHarness{
		Seed:     seed,
		Failures: make(map[string]int),
	}
}

func (h *CrashInjectionHarness) RecordFailure(ft FailureType) {
	h.mu.Lock()
	defer h.mu.Unlock()
	h.Failures[string(ft)]++
}

func (h *CrashInjectionHarness) Count(ft FailureType) int {
	h.mu.Lock()
	defer h.mu.Unlock()
	return h.Failures[string(ft)]
}

func (h *CrashInjectionHarness) TotalFailures() int {
	h.mu.Lock()
	defer h.mu.Unlock()
	total := 0
	for _, c := range h.Failures {
		total += c
	}
	return total
}

// 1. PanicRecoveryPath: spawns a goroutine that panics and recovers
func (h *CrashInjectionHarness) PanicRecoveryPath(shouldPanic bool) (err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("recovered from panic: %v", r)
			h.RecordFailure(PanicRecovery)
		}
	}()

	if shouldPanic {
		panic("simulated crash: worker panic")
	}
	return nil
}

// 2. OOMPath: simulates memory exhaustion by rejecting allocation
func (h *CrashInjectionHarness) OOMPath(simulate bool) error {
	if simulate {
		h.RecordFailure(OOMPath)
		return fmt.Errorf("OOM: memory allocation failed for buffer of size 1048576")
	}
	return nil
}

// 3. NetworkPartitionPath: simulates network unavailability
func (h *CrashInjectionHarness) NetworkPartitionPath(simulate bool, target string) error {
	if simulate {
		h.RecordFailure(NetworkPartition)
		return fmt.Errorf("network partition: cannot reach %s (connection refused)", target)
	}
	return nil
}

// 4. DeadlockDetectionPath: detects potential deadlocks via timeout
func (h *CrashInjectionHarness) DeadlockDetectionPath(ctx context.Context, timeout time.Duration) error {
	done := make(chan struct{})
	go func() {
		// Simulate work that might deadlock
		time.Sleep(50 * time.Millisecond)
		close(done)
	}()

	select {
	case <-done:
		return nil
	case <-ctx.Done():
		h.RecordFailure(DeadlockDetection)
		return fmt.Errorf("deadlock detected: worker timed out after %v", timeout)
	case <-time.After(timeout):
		h.RecordFailure(DeadlockDetection)
		return fmt.Errorf("deadlock detected: operation timed out after %v", timeout)
	}
}

// 5. SchedulerStallPath: simulates scheduler not making progress
func (h *CrashInjectionHarness) SchedulerStallPath(simulate bool, stallDuration time.Duration) error {
	if simulate {
		// Detect stall by measuring actual vs expected duration
		start := time.Now()
		// Simulate a goroutine that doesn't yield
		done := make(chan bool)
		go func() {
			// Busy loop simulating scheduler stall
			for time.Since(start) < stallDuration {
			}
			done <- true
		}()
		<-done

		elapsed := time.Since(start)
		if elapsed >= stallDuration {
			h.RecordFailure(SchedulerStall)
			return fmt.Errorf("scheduler stall detected: expected %v, actual %v", stallDuration, elapsed)
		}
	}
	return nil
}

// 6. ResourceExhaustionPath: simulates resource pool exhaustion
func (h *CrashInjectionHarness) ResourceExhaustionPath(simulate bool, resource string, poolSize int) error {
	if simulate {
		h.RecordFailure(ResourceExhaustion)
		return fmt.Errorf("resource exhaustion: %s pool exhausted (max %d connections, all in use)", resource, poolSize)
	}
	return nil
}

// RunDeterministicTesting runs a set of test scenarios with deterministic failure injection.
// Returns a report of all failure modes triggered.
func (h *CrashInjectionHarness) RunDeterministicTesting(seed int64, scenarios []FailureType) map[string]int {
	rng := rand.New(rand.NewSource(seed))
	results := make(map[string]int)

	for _, s := range scenarios {
		shouldFail := rng.Intn(2) == 0
		switch s {
		case PanicRecovery:
			h.PanicRecoveryPath(shouldFail)
		case OOMPath:
			h.OOMPath(shouldFail)
		case NetworkPartition:
			h.NetworkPartitionPath(shouldFail, fmt.Sprintf("node-%d", rng.Intn(100)))
		case SchedulerStall:
			h.SchedulerStallPath(shouldFail, 10*time.Millisecond)
		case ResourceExhaustion:
			h.ResourceExhaustionPath(shouldFail, "goroutine", 100)
		}
		results[string(s)] = h.Failures[string(s)]
	}
	return results
}