@@ -28,6 +28,12 @@ const (
2828 restartBackoff = time .Second // base crash-restart delay (doubles, capped)
2929 maxBackoff = 30 * time .Second
3030 healthyUptime = 30 * time .Second // a run longer than this resets the backoff
31+
32+ // Wedged-process watchdog: the exit monitor restarts a process that DIES; this
33+ // covers one that stays alive but stops serving (Xray's API stops answering).
34+ watchdogInterval = 30 * time .Second // how often to probe a running process
35+ watchdogFailsToAct = 3 // consecutive failed probes before restarting (~90s wedged)
36+ watchdogCooldown = 5 * time .Minute // min gap between watchdog restarts (anti-storm)
3137)
3238
3339// proc is a single running Xray child. done is closed once Wait() has reaped it;
@@ -70,8 +76,16 @@ type Supervisor struct {
7076 suspended bool
7177 restarting bool // a deliberate stop→start is in flight (the ~1s bounce)
7278
79+ // lastWatchdog is when the wedged-process watchdog last restarted Xray, for its
80+ // anti-storm cooldown (zero until it fires).
81+ lastWatchdog time.Time
82+ // probe reports whether Xray is answering its API; defaults to apiResponsive and is
83+ // swappable in tests so the watchdog decision can be exercised without a live Xray.
84+ probe func () bool
85+
7386 onAccess func (email , ip , dest string ) // called per access-log connection line
7487 onCrash func (err error ) // called when Xray exits unexpectedly (crash)
88+ onWedged func () // called when the watchdog restarts a wedged process
7589 // onRecover is called when a SUPERVISED restart succeeds — i.e. Xray is back up
7690 // after a crash. Deliberately not fired by Apply-driven restarts (a reconcile, a
7791 // renewed certificate): those are routine, and reporting them as recovery would
@@ -143,6 +157,100 @@ func (s *Supervisor) SetOnCrash(fn func(err error)) { s.onCrash = fn }
143157// SetOnRecover registers a callback invoked when Xray comes back after a crash.
144158func (s * Supervisor ) SetOnRecover (fn func ()) { s .onRecover = fn }
145159
160+ // SetOnWedged registers a callback invoked when the watchdog restarts a wedged
161+ // process (alive but no longer serving). Used to alert the operator — this is an
162+ // outage the crash path never reported, because the process never exited.
163+ func (s * Supervisor ) SetOnWedged (fn func ()) { s .onWedged = fn }
164+
165+ // StartWatchdog launches the wedged-process watchdog in the background: it probes a
166+ // running Xray and, if the process is alive but its API has stopped answering for
167+ // several checks in a row, restarts it. This is the gap the exit monitor cannot
168+ // cover — that one only sees a process that DIES. Idempotent-ish (call once); a
169+ // missing binary makes it a no-op. The loop exits when the supervisor is Stopped.
170+ func (s * Supervisor ) StartWatchdog () {
171+ if s .bin == "" {
172+ return
173+ }
174+ go s .watchdogLoop ()
175+ }
176+
177+ func (s * Supervisor ) watchdogLoop () {
178+ if s .probe == nil {
179+ s .probe = s .apiResponsive
180+ }
181+ t := time .NewTicker (watchdogInterval )
182+ defer t .Stop ()
183+ fails := 0
184+ for range t .C {
185+ s .mu .Lock ()
186+ closed := s .closed
187+ s .mu .Unlock ()
188+ if closed {
189+ return
190+ }
191+ var act bool
192+ fails , act = s .watchdogTick (fails )
193+ if ! act {
194+ continue
195+ }
196+ slog .Error ("xray watchdog: wedged (alive but not serving) — restarting" )
197+ if s .onWedged != nil {
198+ go s .onWedged ()
199+ }
200+ if err := s .Restart (); err != nil {
201+ slog .Error ("xray watchdog: restart failed" , "err" , err )
202+ }
203+ }
204+ }
205+
206+ // watchdogTick evaluates one probe cycle and returns the updated consecutive-failure
207+ // count and whether a wedged process should be restarted now. The decision, minus the
208+ // ticker and the restart itself, so it is unit-testable without a live Xray:
209+ // - a down / suspended / mid-bounce supervisor is never judged (a routine restart is
210+ // not a wedge), and resets the counter;
211+ // - a responsive process resets the counter;
212+ // - only after watchdogFailsToAct failures in a row, and past the cooldown since the
213+ // last watchdog restart, does it say to act (recording the restart time).
214+ func (s * Supervisor ) watchdogTick (fails int ) (int , bool ) {
215+ s .mu .Lock ()
216+ watch := s .cur != nil && ! s .suspended && ! s .restarting
217+ s .mu .Unlock ()
218+ if ! watch {
219+ return 0 , false
220+ }
221+ probe := s .probe
222+ if probe == nil {
223+ probe = s .apiResponsive
224+ }
225+ if probe () {
226+ return 0 , false
227+ }
228+ fails ++
229+ if fails < watchdogFailsToAct {
230+ slog .Warn ("xray watchdog: process alive but not answering its API" , "fails" , fails )
231+ return fails , false
232+ }
233+ // Wedged for watchdogFailsToAct probes in a row. Honour a cooldown so a process
234+ // that wedges again right after a restart can't spin us into a restart storm.
235+ s .mu .Lock ()
236+ cooling := ! s .lastWatchdog .IsZero () && time .Since (s .lastWatchdog ) < watchdogCooldown
237+ if ! cooling {
238+ s .lastWatchdog = time .Now ()
239+ }
240+ s .mu .Unlock ()
241+ if cooling {
242+ return fails , false // still wedged; hold off until the cooldown elapses
243+ }
244+ return 0 , true
245+ }
246+
247+ // apiResponsive reports whether the running Xray still answers its API — a failed,
248+ // timeout-bounded stats query is the "wedged" signal the exit monitor never sees.
249+ func (s * Supervisor ) apiResponsive () bool {
250+ _ , err := s .QueryStats (s .APIAddr ())
251+ return err == nil
252+ }
253+
146254// recovered fires the recovery callback off the restart path, mirroring onCrash.
147255func (s * Supervisor ) recovered () {
148256 if s .onRecover != nil {
0 commit comments