fix: reap zombie processes

Author: 54rt1n

src/app/process.go

@@ -187,6 +187,14 @@ func (p *Process) waitForStdOutErr() {
 	ctx, cancel := context.WithCancel(context.Background())
 	if p.procConf.IsDaemon {
 		ctx, cancel = context.WithTimeout(context.Background(), time.Duration(p.procConf.LaunchTimeout)*time.Second)
+	} else if p.isState(types.ProcessStateTerminating) {
+		// In terminating state, never block forever on stdout/stderr drain.
+		// Reaching command.Wait() is required to reap child processes.
+		timeoutSec := p.procConf.ShutDownParams.ShutDownTimeout
+		if timeoutSec == UndefinedShutdownTimeoutSec {
+			timeoutSec = DefaultShutdownTimeoutSec
+		}
+		ctx, cancel = context.WithTimeout(context.Background(), time.Duration(timeoutSec)*time.Second)
 	}
 	defer cancel()
 	if p.stdOutDone != nil {

src/app/project_runner.go

@@ -195,6 +195,20 @@ func (p *ProjectRunner) runProcess(config *types.ProcessConfig) {
 		procLog = pclog.NewLogBuffer(0)
 	}
 	procState, _ := p.GetProcessState(config.ReplicaName)
+	// Recover from stale "Terminating" state snapshots (for example when a
+	// process ended as zombie and never transitioned to Completed). Reusing
+	// that snapshot would cause Process.run() to short-circuit immediately.
+	if procState != nil &&
+		procState.Status == types.ProcessStateTerminating &&
+		!procState.IsRunning {
+		log.Warn().
+			Str("process", config.ReplicaName).
+			Msg("Resetting stale terminating state before start")
+		procState = types.NewProcessState(config)
+		p.statesMutex.Lock()
+		p.processStates[config.ReplicaName] = procState
+		p.statesMutex.Unlock()
+	}
 	isMain := config.Name == p.mainProcess
 	hasMain := p.mainProcess != ""
 	printLogs := !hasMain && !p.isTuiOn && !p.project.MCPServer.IsStdio()

src/app/system_test.go

@@ -1431,6 +1431,83 @@ func TestSystem_ConcurrentRestartRaceCondition(t *testing.T) {
 	}
 }
 
+func TestSystem_StartProcessResetsStaleTerminatingState(t *testing.T) {
+	testProcess := "stale_terminating"
+	shell := command.DefaultShellConfig()
+
+	project := &types.Project{
+		Processes: map[string]types.ProcessConfig{
+			testProcess: {
+				Name:        testProcess,
+				ReplicaName: testProcess,
+				Executable:  shell.ShellCommand,
+				Args:        []string{shell.ShellArgument, getSleepCommand(1.0)},
+				RestartPolicy: types.RestartPolicyConfig{
+					Restart: types.RestartPolicyNo,
+				},
+			},
+		},
+		ShellConfig: shell,
+	}
+
+	runner, err := NewProjectRunner(&ProjectOpts{
+		project:         project,
+		processesToRun:  []string{},
+		noDeps:          false,
+		mainProcess:     "",
+		mainProcessArgs: []string{},
+		isTuiOn:         false,
+	})
+	if err != nil {
+		t.Error(err.Error())
+		return
+	}
+
+	// Simulate stale state from a prior broken termination.
+	runner.statesMutex.Lock()
+	runner.processStates[testProcess].Status = types.ProcessStateTerminating
+	runner.processStates[testProcess].IsRunning = false
+	runner.statesMutex.Unlock()
+	runner.runProcMutex.Lock()
+	runner.runningProcesses = make(map[string]*Process)
+	runner.runProcMutex.Unlock()
+	runner.doneProcMutex.Lock()
+	runner.doneProcesses = make(map[string]*Process)
+	runner.doneProcMutex.Unlock()
+	runner.logger = pclog.NewNilLogger()
+
+	if err := runner.StartProcess(testProcess); err != nil {
+		t.Fatalf("failed to start process: %v", err)
+	}
+
+	var lastStatus string
+	for attempts := range 200 {
+		state, stateErr := runner.GetProcessState(testProcess)
+		if stateErr != nil {
+			t.Fatalf("failed to get process state: %v", stateErr)
+		}
+		lastStatus = state.Status
+		if state.Status == types.ProcessStateRunning || state.Status == types.ProcessStateCompleted {
+			break
+		}
+		time.Sleep(10 * time.Millisecond)
+		if attempts == 199 {
+			t.Fatalf("process failed to leave stale Terminating state, last status=%s", state.Status)
+		}
+	}
+
+	if lastStatus == types.ProcessStateTerminating {
+		t.Fatalf("expected process to recover from stale Terminating state, got %s", lastStatus)
+	}
+
+	// Cleanup if still running.
+	if runner.getRunningProcess(testProcess) != nil {
+		if err := runner.StopProcess(testProcess); err != nil {
+			t.Fatalf("failed to stop process: %v", err)
+		}
+	}
+}
+
 func TestReadinessProbeRestart(t *testing.T) {
 	proc := &types.ProcessConfig{
 		Name:        "test",