fix(control-plane): resolve agent node health status flapping

control-plane/internal/config/config.go

@@ -4,6 +4,7 @@ import (
 	"fmt"           // Added for fmt.Errorf
 	"os"            // Added for os.Stat, os.ReadFile
 	"path/filepath" // Added for filepath.Join
+	"strconv"
 	"time"
 
 	"gopkg.in/yaml.v3" // Added for yaml.Unmarshal
@@ -32,10 +33,21 @@ type UIConfig struct {
 // AgentFieldConfig holds the core AgentField server configuration.
 type AgentFieldConfig struct {
 	Port             int                    `yaml:"port"`
+	NodeHealth       NodeHealthConfig       `yaml:"node_health" mapstructure:"node_health"`
 	ExecutionCleanup ExecutionCleanupConfig `yaml:"execution_cleanup" mapstructure:"execution_cleanup"`
 	ExecutionQueue   ExecutionQueueConfig   `yaml:"execution_queue" mapstructure:"execution_queue"`
 }
 
+// NodeHealthConfig holds configuration for agent node health monitoring.
+// Zero values are treated as "use default" — set explicitly to override.
+type NodeHealthConfig struct {
+	CheckInterval           time.Duration `yaml:"check_interval" mapstructure:"check_interval"`                       // How often to HTTP health check nodes (0 = default 10s)
+	CheckTimeout            time.Duration `yaml:"check_timeout" mapstructure:"check_timeout"`                         // Timeout per HTTP health check (0 = default 5s)
+	ConsecutiveFailures     int           `yaml:"consecutive_failures" mapstructure:"consecutive_failures"`            // Failures before marking inactive (0 = default 3; set 1 for instant)
+	RecoveryDebounce        time.Duration `yaml:"recovery_debounce" mapstructure:"recovery_debounce"`                 // Wait before allowing inactive->active (0 = default 5s)
+	HeartbeatStaleThreshold time.Duration `yaml:"heartbeat_stale_threshold" mapstructure:"heartbeat_stale_threshold"` // Heartbeat age before marking stale (0 = default 60s)
+}
+
 // ExecutionCleanupConfig holds configuration for execution cleanup and garbage collection
 type ExecutionCleanupConfig struct {
 	Enabled                bool          `yaml:"enabled" mapstructure:"enabled" default:"true"`
@@ -170,4 +182,31 @@ func applyEnvOverrides(cfg *Config) {
 	if apiKey := os.Getenv("AGENTFIELD_API_AUTH_API_KEY"); apiKey != "" {
 		cfg.API.Auth.APIKey = apiKey
 	}
+
+	// Node health monitoring overrides
+	if val := os.Getenv("AGENTFIELD_HEALTH_CHECK_INTERVAL"); val != "" {
+		if d, err := time.ParseDuration(val); err == nil {
+			cfg.AgentField.NodeHealth.CheckInterval = d
+		}
+	}
+	if val := os.Getenv("AGENTFIELD_HEALTH_CHECK_TIMEOUT"); val != "" {
+		if d, err := time.ParseDuration(val); err == nil {
+			cfg.AgentField.NodeHealth.CheckTimeout = d
+		}
+	}
+	if val := os.Getenv("AGENTFIELD_HEALTH_CONSECUTIVE_FAILURES"); val != "" {
+		if i, err := strconv.Atoi(val); err == nil {
+			cfg.AgentField.NodeHealth.ConsecutiveFailures = i
+		}
+	}
+	if val := os.Getenv("AGENTFIELD_HEALTH_RECOVERY_DEBOUNCE"); val != "" {
+		if d, err := time.ParseDuration(val); err == nil {
+			cfg.AgentField.NodeHealth.RecoveryDebounce = d
+		}
+	}
+	if val := os.Getenv("AGENTFIELD_HEARTBEAT_STALE_THRESHOLD"); val != "" {
+		if d, err := time.ParseDuration(val); err == nil {
+			cfg.AgentField.NodeHealth.HeartbeatStaleThreshold = d
+		}
+	}
 }

control-plane/internal/handlers/nodes.go

@@ -299,9 +299,10 @@ var (
 	heartbeatCache = &HeartbeatCache{
 		nodes: make(map[string]*CachedNodeData),
 	}
-	// Only write to DB if heartbeat is older than this threshold
-	// Must be less than health monitor grace period (12s) to prevent false inactivity
-	dbUpdateThreshold = 8 * time.Second
+	// Only write to DB if heartbeat is older than this threshold.
+	// Reduced from 8s to 2s to keep DB timestamps fresh and prevent
+	// other systems (reconciliation, health monitor) from seeing stale data.
+	dbUpdateThreshold = 2 * time.Second
 )
 
 // shouldUpdateDatabase determines if a heartbeat should trigger a database update

control-plane/internal/server/server.go

@@ -116,9 +116,10 @@ func NewAgentFieldServer(cfg *config.Config) (*AgentFieldServer, error) {
 
 	// Initialize StatusManager for unified status management
 	statusManagerConfig := services.StatusManagerConfig{
-		ReconcileInterval: 30 * time.Second,
-		StatusCacheTTL:    5 * time.Minute,
-		MaxTransitionTime: 2 * time.Minute,
+		ReconcileInterval:       30 * time.Second,
+		StatusCacheTTL:          5 * time.Minute,
+		MaxTransitionTime:       2 * time.Minute,
+		HeartbeatStaleThreshold: cfg.AgentField.NodeHealth.HeartbeatStaleThreshold,
 	}
 
 	// Create UIService first (without StatusManager)
@@ -140,8 +141,13 @@ func NewAgentFieldServer(cfg *config.Config) (*AgentFieldServer, error) {
 
 	executionsUIService := services.NewExecutionsUIService(storageProvider) // Initialize ExecutionsUIService
 
-	// Initialize health monitor with StatusManager integration
-	healthMonitorConfig := services.HealthMonitorConfig{}
+	// Initialize health monitor with configurable settings
+	healthMonitorConfig := services.HealthMonitorConfig{
+		CheckInterval:       cfg.AgentField.NodeHealth.CheckInterval,
+		CheckTimeout:        cfg.AgentField.NodeHealth.CheckTimeout,
+		ConsecutiveFailures: cfg.AgentField.NodeHealth.ConsecutiveFailures,
+		RecoveryDebounce:    cfg.AgentField.NodeHealth.RecoveryDebounce,
+	}
 	healthMonitor := services.NewHealthMonitor(storageProvider, healthMonitorConfig, uiService, agentClient, statusManager, presenceManager)
 	presenceManager.SetExpireCallback(healthMonitor.UnregisterAgent)