fix(af run): eliminate port race that killed healthy nodes on startup (#737)

`af run <node>` intermittently failed with 'agent node did not become ready
within 10s' — worst for import-heavy nodes like pr-af. Root cause is a
check-then-exec race between the runner and the SDK:

1. The runner probes a free port, releases the probe, exports PORT=N, then
   polls ONLY port N for readiness.
2. If N is momentarily unavailable when the child binds, the SDK SILENTLY
   moves to the next free port (N+1) — a port the runner never learns about.
   The runner then polls the dead port N until timeout and kills a node that
   actually came up fine on N+1.
3. The 10s readiness window is also too short for nodes with large import
   graphs, independent of the race.

Fix (regression-safe, gated):
- Runner exports AGENTFIELD_STRICT_PORT=1 alongside PORT.
- SDK, only when that signal is set, binds the injected PORT authoritatively:
  if it is unavailable it exits with a clear error instead of silently
  bumping, so the runner and node can never disagree about the port. Absent
  the signal (standalone `python -m <node>.app`, manual PORT=...), the old
  lenient auto-bump behavior is unchanged — no regression.
- Readiness timeout is now 30s and configurable via
  AGENTFIELD_NODE_READY_TIMEOUT (whole seconds).
- Guard PortManager's reserved-ports map with a mutex (it was read/written
  without synchronization).

Verified: with the patched control plane + SDK, all four reference nodes
(swe-planner, cloudsecurity-af, sec-af, pr-af) start on their assigned ports
and pass health; pr-af — previously failing almost every run — now binds its
assigned port every time.

Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

Author: AbirAbbas

control-plane/internal/core/services/agent_service.go

@@ -11,6 +11,7 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
+	"strconv"
 	"strings"
 	"syscall"
 	"time"
@@ -120,7 +121,7 @@ func (as *DefaultAgentService) runAgentGuarded(name string, options domain.RunOp
 	if metadata, err := packages.ParsePackageMetadata(agentNode.Path); err == nil {
 		healthPath = metadata.HealthcheckPath()
 	}
-	if err := as.waitForAgentNode(port, healthPath, 10*time.Second); err != nil {
+	if err := as.waitForAgentNode(port, healthPath, nodeReadyTimeout()); err != nil {
 		// Kill the process if it failed to start properly
 		if stopErr := as.processManager.Stop(pid); stopErr != nil {
 			return nil, fmt.Errorf("agent node failed to start: %w (additionally failed to stop process: %v)", err, stopErr)
@@ -513,6 +514,11 @@ func (as *DefaultAgentService) buildProcessConfig(agentNode packages.InstalledPa
 	serverURL := resolveServerURL()
 	env := os.Environ()
 	env = append(env, fmt.Sprintf("PORT=%d", port))
+	// Tell the SDK to bind exactly this port and fail fast if it is unavailable,
+	// rather than silently moving to another port that the runner is not polling
+	// (the readiness check below targets this exact port). Gated by this signal so
+	// standalone `python -m <node>.app` keeps its lenient auto-port fallback.
+	env = append(env, "AGENTFIELD_STRICT_PORT=1")
 	env = append(env, fmt.Sprintf("AGENTFIELD_SERVER=%s", serverURL))
 	env = append(env, fmt.Sprintf("AGENTFIELD_SERVER_URL=%s", serverURL))
 
@@ -616,6 +622,20 @@ func (as *DefaultAgentService) resolveNodeEnvironment(nodeName string, metadata
 	return resolver.Resolve(env)
 }
 
+// nodeReadyTimeout is how long to wait for a freshly started node to answer its
+// health check. Import-heavy nodes (large dependency graphs) routinely take more
+// than the old hardcoded 10s to boot, which produced spurious "did not become
+// ready" failures on nodes that were actually starting fine. Default 30s,
+// overridable via AGENTFIELD_NODE_READY_TIMEOUT (whole seconds).
+func nodeReadyTimeout() time.Duration {
+	if v := os.Getenv("AGENTFIELD_NODE_READY_TIMEOUT"); v != "" {
+		if secs, err := strconv.Atoi(v); err == nil && secs > 0 {
+			return time.Duration(secs) * time.Second
+		}
+	}
+	return 30 * time.Second
+}
+
 // waitForAgentNode waits for the agent node to become ready
 func (as *DefaultAgentService) waitForAgentNode(port int, healthPath string, timeout time.Duration) error {
 	if healthPath == "" {

control-plane/internal/infrastructure/process/port_manager.go

@@ -4,13 +4,15 @@ package process
 import (
 	"fmt"
 	"net"
+	"sync"
 
 	"github.com/Agent-Field/agentfield/control-plane/internal/core/interfaces"
 )
 
 // DefaultPortManager provides a default implementation for managing network ports.
 // It keeps track of reserved ports in memory.
 type DefaultPortManager struct {
+	mu            sync.Mutex
 	reservedPorts map[int]bool
 }
 
@@ -26,10 +28,10 @@ func NewPortManager() interfaces.PortManager {
 // It checks both system availability and internal reservations.
 // Returns the first free port found, or an error if no port is available in the range.
 func (pm *DefaultPortManager) FindFreePort(startPort int) (int, error) {
-	// The search range is typically small, e.g., 100 ports.
-	// This can be made configurable if needed.
+	pm.mu.Lock()
+	defer pm.mu.Unlock()
 	for port := startPort; port <= startPort+100; port++ {
-		if pm.IsPortAvailable(port) && !pm.reservedPorts[port] {
+		if !pm.reservedPorts[port] && pm.IsPortAvailable(port) {
 			return port, nil
 		}
 	}
@@ -53,6 +55,8 @@ func (pm *DefaultPortManager) IsPortAvailable(port int) bool {
 // It first checks if the port is system-available before reserving.
 // Returns an error if the port is not available or cannot be reserved.
 func (pm *DefaultPortManager) ReservePort(port int) error {
+	pm.mu.Lock()
+	defer pm.mu.Unlock()
 	if !pm.IsPortAvailable(port) {
 		return fmt.Errorf("port %d is not available at the system level", port)
 	}
@@ -67,6 +71,8 @@ func (pm *DefaultPortManager) ReservePort(port int) error {
 // This makes the port available for future reservations by this manager.
 // It does not affect the system-level availability of the port.
 func (pm *DefaultPortManager) ReleasePort(port int) error {
+	pm.mu.Lock()
+	defer pm.mu.Unlock()
 	if _, ok := pm.reservedPorts[port]; !ok {
 		return fmt.Errorf("port %d was not reserved by this manager", port)
 	}

sdk/python/agentfield/agent_server.py

@@ -677,7 +677,26 @@ def serve(
             env_port = os.getenv("PORT")
             if env_port and env_port.isdigit():
                 suggested_port = int(env_port)
-                if AgentUtils.is_port_available(suggested_port):
+                if os.getenv("AGENTFIELD_STRICT_PORT") == "1":
+                    # The AgentField runner assigned this exact port and polls it
+                    # for readiness. Bind it authoritatively — never silently move
+                    # to another port, or the runner would poll a port nothing is
+                    # listening on and kill us for "not becoming ready". If it is
+                    # genuinely unavailable, exit fast with a clear error so the
+                    # runner surfaces a real failure instead of a phantom timeout.
+                    if not AgentUtils.is_port_available(suggested_port):
+                        log_error(
+                            f"AGENTFIELD_STRICT_PORT set but the assigned port "
+                            f"{suggested_port} is unavailable; exiting so the "
+                            f"control plane can reallocate and retry"
+                        )
+                        raise RuntimeError(
+                            f"assigned port {suggested_port} is unavailable"
+                        )
+                    port = suggested_port
+                    if self.agent.dev_mode:
+                        log_debug(f"Using assigned port from AgentField CLI: {port}")
+                elif AgentUtils.is_port_available(suggested_port):
                     port = suggested_port
                     if self.agent.dev_mode:
                         log_debug(f"Using port from AgentField CLI: {port}")