windows: wait for shutdown teardown to finish before main returns

kardianos/service v1.2.1's windowsService.Run returns as soon as SCM STOP
is processed, which lets main return and the Go runtime call ExitProcess
before otelcol.Shutdown (running on a parallel goroutine) can complete.
In the pristine change this rarely mattered because Shutdown is intrinsically
sub-second, but a slow-shutdown plugin, a network flush, or any future
teardown work could be truncated -- the final "Shutdown complete." log
line and any pending flushes would be dropped without any indication.

Add a done-channel signalled by (*program).run and waited on by main
after s.Run() returns:

  - shutdown_signal.go: runCompleteChan (chan struct{}) plus sync.Once
    guard, runCompleteTimeout = 30s, signalRunComplete(), waitRunComplete().
  - amazon-cloudwatch-agent.go: (*program).run defers signalRunComplete
    so it fires on every code path, including the Windows fallback where
    handleTerminatingSignal calls os.Exit. main's Windows service branch
    waits on runCompleteChan after s.Run returns.

kardianos still reports SERVICE_STOPPED to SCM as soon as prg.Stop
returns, so the OS shutdown of other services proceeds in parallel.
Only this process is held back, and only until its own teardown finishes
-- capped at runCompleteTimeout so a stuck teardown cannot indefinitely
hold up the OS.

Tests: three new unit tests (idempotent signal, wait-then-signal,
wait-then-timeout). All 47 existing + new tests pass.

Empirically validated on Windows Server 2022: with a 10-second sleep
injected inside otelcol Service.Shutdown, the earlier change loses the
"Shutdown complete." log line entirely (main returns before Shutdown
finishes), whereas with this wait the line is preserved and the full
teardown sequence completes. No 6008 / Kernel-Power 41; total OS
shutdown +10s (matching the injected delay).

Author: Kenta Hayashi

cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go

@@ -435,6 +435,7 @@ func (p *program) Start(_ service.Service) error {
 	return nil
 }
 func (p *program) run() {
+	defer signalRunComplete() // unblock main()'s waitRunComplete on any return
 	stop = make(chan struct{})
 	reloadLoop(
 		stop,
@@ -649,6 +650,9 @@ func main() {
 			if err != nil {
 				log.Println("E! " + err.Error())
 			}
+			// Wait for (*program).run to finish so otelcol.Shutdown can
+			// complete before the Go runtime calls ExitProcess.
+			waitRunComplete()
 		}
 	} else {
 		stop = make(chan struct{})

cmd/amazon-cloudwatch-agent/shutdown_signal.go

@@ -5,6 +5,7 @@ package main
 
 import (
 	"log"
+	"sync"
 	"sync/atomic"
 	"time"
 )
@@ -40,3 +41,32 @@ func handleTerminatingSignalDispatch(stopCh <-chan struct{}, timeout time.Durati
 		terminatingSignalReceived.Store(true)
 	}
 }
+
+// runCompleteChan is closed by signalRunComplete when (*program).run
+// finishes. main() waits on this via waitRunComplete so otelcol.Shutdown
+// and any pending flushes can complete before the process exits.
+// Declared as var so tests can substitute a fresh channel.
+var runCompleteChan = make(chan struct{})
+
+// runCompleteOnce guards close(runCompleteChan).
+var runCompleteOnce sync.Once
+
+// runCompleteTimeout bounds waitRunComplete so a stuck teardown cannot
+// indefinitely hold up the OS.
+var runCompleteTimeout = 30 * time.Second
+
+// signalRunComplete unblocks waitRunComplete. Safe to call multiple times.
+func signalRunComplete() {
+	runCompleteOnce.Do(func() { close(runCompleteChan) })
+}
+
+// waitRunComplete blocks until signalRunComplete or runCompleteTimeout,
+// whichever comes first. Logs a warning on timeout so the OS is not
+// held indefinitely.
+func waitRunComplete() {
+	select {
+	case <-runCompleteChan:
+	case <-time.After(runCompleteTimeout):
+		log.Printf("W! Windows service: waited %v for shutdown teardown to complete; exiting anyway", runCompleteTimeout)
+	}
+}

cmd/amazon-cloudwatch-agent/shutdown_signal_test.go

@@ -95,3 +95,69 @@ func TestDispatch_SCMAcceptedButTimeout_SetsFallbackFlag(t *testing.T) {
 		t.Fatal("expected fallback flag set after SCM path timeout")
 	}
 }
+
+// ---- runComplete channel + wait -------------------------------------
+
+// resetRunComplete restores a fresh channel + Once so tests can exercise
+// signalRunComplete and waitRunComplete in isolation.
+func resetRunComplete() {
+	runCompleteChan = make(chan struct{})
+	runCompleteOnce = sync.Once{}
+}
+
+// signalRunComplete must be idempotent (close of a closed channel would
+// panic without sync.Once).
+func TestSignalRunComplete_Idempotent(t *testing.T) {
+	t.Cleanup(resetRunComplete)
+	resetRunComplete()
+
+	signalRunComplete()
+	signalRunComplete() // must not panic
+
+	select {
+	case <-runCompleteChan:
+	default:
+		t.Fatal("runCompleteChan not closed after signalRunComplete()")
+	}
+}
+
+// waitRunComplete must return promptly once signalRunComplete is called.
+func TestWaitRunComplete_ReturnsWhenSignaled(t *testing.T) {
+	t.Cleanup(resetRunComplete)
+	resetRunComplete()
+	oldTimeout := runCompleteTimeout
+	runCompleteTimeout = 500 * time.Millisecond
+	t.Cleanup(func() { runCompleteTimeout = oldTimeout })
+
+	go func() {
+		time.Sleep(10 * time.Millisecond)
+		signalRunComplete()
+	}()
+
+	start := time.Now()
+	waitRunComplete()
+	elapsed := time.Since(start)
+	if elapsed > 200*time.Millisecond {
+		t.Fatalf("waitRunComplete took %v; expected quick return after signal", elapsed)
+	}
+}
+
+// waitRunComplete must fall through after runCompleteTimeout when no
+// signal arrives, so the OS is not held indefinitely.
+func TestWaitRunComplete_TimesOut(t *testing.T) {
+	t.Cleanup(resetRunComplete)
+	resetRunComplete()
+	oldTimeout := runCompleteTimeout
+	runCompleteTimeout = 20 * time.Millisecond
+	t.Cleanup(func() { runCompleteTimeout = oldTimeout })
+
+	start := time.Now()
+	waitRunComplete()
+	elapsed := time.Since(start)
+	if elapsed < 15*time.Millisecond {
+		t.Fatalf("waitRunComplete returned in %v; expected ~20ms timeout", elapsed)
+	}
+	if elapsed > 200*time.Millisecond {
+		t.Fatalf("waitRunComplete took %v; expected ~20ms timeout", elapsed)
+	}
+}