windows: route OS-shutdown SIGTERM through SCM to unblock svc.Run

At Windows OS shutdown, when the collector runs as a service child
(spawned by start-amazon-cloudwatch-agent.exe), csrss delivers
CTRL_SHUTDOWN (mapped to SIGTERM) to the collector but the SCM routes
SERVICE_CONTROL_SHUTDOWN through the launcher, not directly here.
kardianos/service v1.2.1's non-interactive windowsService.Run waits only
on the SCM control channel and does not watch SIGTERM, so svc.Run never
returns and the whole OS shutdown deadlocks until the platform's
hard-timeout (~4-5 min: Event ID 6008 + Kernel-Power 41). sc stop /
Stop-Service is unaffected.

On a non-SIGHUP terminating OS signal, reloadLoop's signal goroutine now
issues SERVICE_CONTROL_STOP against this process's own SCM entry (found
by matching ProcessId, since kardianos's config Name -- 'telegraf' by
default -- is not registered in SCM) via x/sys/windows/svc/mgr. That
triggers kardianos's normal STOP path (prg.Stop -> close(stop)) and
svc.Run returns cleanly, letting main return and OS shutdown proceed.

If the SCM path is unavailable (not running as a Windows service,
Connect/ListServices/OpenService/Control fails, or prg.Stop doesn't run
within stopWaitTimeout=30s), the code falls back to setting a flag that
handleTerminatingSignal reads after reloadLoop returns to call
os.Exit(0) as a last resort so the OS shutdown is never blocked.

No-op on non-Windows, in interactive/console mode, on the SCM STOP path
(<-stop), and on SIGHUP reload. No new module dependencies (x/sys is
already in go.mod). Tests cover the atomic flag, the dispatch function
(cross-platform, via test seam), and the Windows-specific SCM logic and
fallback exit (via fake scmManager/scmService and seams over exitFunc /
isWinService / ownProcessID / scmConnectFunc).

Validated on Windows Server 2022 (EC2 m5.xlarge, WaitToKillServiceTimeout=300000):
baseline OS shutdown hung ~290s + 6008/41; with this patch the collector
is stopped via its own SCM entry in ~34s (Event ID 6006, no 6008/41).
sc stop still ~0.3s.

Author: Kenta Hayashi

cmd/amazon-cloudwatch-agent/amazon-cloudwatch-agent.go

@@ -126,6 +126,10 @@ func reloadLoop(
 					log.Println("I! Reloading Telegraf config")
 					<-reload
 					reload <- true
+				} else {
+					// Route non-SIGHUP terminating signal through the SCM;
+					// see shutdown_signal_windows.go.
+					handleTerminatingSignalDispatch(stop, stopWaitTimeout)
 				}
 				cancel()
 			case <-stop:
@@ -439,6 +443,8 @@ func (p *program) run() {
 		p.aggregatorFilters,
 		p.processorFilters,
 	)
+	// Windows-only fallback: exit if SCM STOP was not taken.
+	handleTerminatingSignal()
 }
 func (p *program) Stop(_ service.Service) error {
 	close(stop)

cmd/amazon-cloudwatch-agent/shutdown_signal.go

@@ -0,0 +1,42 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"log"
+	"sync/atomic"
+	"time"
+)
+
+// See shutdown_signal_windows.go for the workaround's purpose.
+
+// stopWaitTimeout bounds how long the signal goroutine waits for prg.Stop
+// after requestSCMStop is accepted. Declared as var for testability.
+var stopWaitTimeout = 30 * time.Second
+
+// terminatingSignalReceived flags that the SCM STOP path was NOT taken;
+// handleTerminatingSignal then os.Exit(0)s so svc.Run cannot block OS
+// shutdown. Accessed atomically.
+var terminatingSignalReceived atomic.Bool
+
+// requestSCMStopFn is a test seam over requestSCMStop.
+var requestSCMStopFn = requestSCMStop
+
+// handleTerminatingSignalDispatch is called by the signal goroutine on a
+// non-SIGHUP terminating signal. Tries SCM STOP; falls back to setting the
+// flag on failure or timeout.
+func handleTerminatingSignalDispatch(stopCh <-chan struct{}, timeout time.Duration) {
+	if requestSCMStopFn() {
+		timer := time.NewTimer(timeout)
+		defer timer.Stop()
+		select {
+		case <-stopCh:
+		case <-timer.C:
+			log.Println("W! Windows service: SCM stop did not deliver in time; forcing fallback")
+			terminatingSignalReceived.Store(true)
+		}
+	} else {
+		terminatingSignalReceived.Store(true)
+	}
+}

cmd/amazon-cloudwatch-agent/shutdown_signal_notwindows.go

@@ -0,0 +1,10 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build !windows
+
+package main
+
+// requestSCMStop and handleTerminatingSignal are no-ops on non-Windows.
+func requestSCMStop() bool     { return false }
+func handleTerminatingSignal() {}

cmd/amazon-cloudwatch-agent/shutdown_signal_test.go

@@ -0,0 +1,97 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"sync"
+	"testing"
+	"time"
+)
+
+// ---- terminatingSignalReceived atomic flag ---------------------------
+
+func TestTerminatingSignalReceived_DefaultUnset(t *testing.T) {
+	t.Cleanup(func() { terminatingSignalReceived.Store(false) })
+	if terminatingSignalReceived.Load() {
+		t.Fatal("terminatingSignalReceived was true before any Store()")
+	}
+}
+
+func TestTerminatingSignalReceived_SetLoad(t *testing.T) {
+	t.Cleanup(func() { terminatingSignalReceived.Store(false) })
+	terminatingSignalReceived.Store(true)
+	if !terminatingSignalReceived.Load() {
+		t.Fatal("terminatingSignalReceived was false after Store(true)")
+	}
+}
+
+func TestTerminatingSignalReceived_Race(t *testing.T) {
+	t.Cleanup(func() { terminatingSignalReceived.Store(false) })
+	const N = 64
+	var wg sync.WaitGroup
+	wg.Add(N)
+	for i := 0; i < N; i++ {
+		go func(i int) {
+			defer wg.Done()
+			if i%2 == 0 {
+				terminatingSignalReceived.Store(true)
+			} else {
+				_ = terminatingSignalReceived.Load()
+			}
+		}(i)
+	}
+	wg.Wait()
+}
+
+// ---- handleTerminatingSignalDispatch ---------------------------------
+
+// withRequestSCMStopFn temporarily overrides requestSCMStopFn and resets
+// the terminatingSignalReceived flag for a test.
+func withRequestSCMStopFn(t *testing.T, fn func() bool) {
+	t.Helper()
+	old := requestSCMStopFn
+	requestSCMStopFn = fn
+	terminatingSignalReceived.Store(false)
+	t.Cleanup(func() {
+		requestSCMStopFn = old
+		terminatingSignalReceived.Store(false)
+	})
+}
+
+// When requestSCMStop returns false (no SCM path), dispatch must set the
+// fallback flag so handleTerminatingSignal can os.Exit later.
+func TestDispatch_SCMUnavailable_SetsFallbackFlag(t *testing.T) {
+	withRequestSCMStopFn(t, func() bool { return false })
+	stopCh := make(chan struct{})
+	handleTerminatingSignalDispatch(stopCh, 5*time.Millisecond)
+	if !terminatingSignalReceived.Load() {
+		t.Fatal("expected fallback flag set when SCM path is unavailable")
+	}
+}
+
+// When requestSCMStop returns true and close(stop) fires before the
+// timeout, dispatch must NOT set the fallback flag (SCM path OK).
+func TestDispatch_SCMSuccess_NoFallbackFlag(t *testing.T) {
+	withRequestSCMStopFn(t, func() bool { return true })
+	stopCh := make(chan struct{})
+	go func() {
+		time.Sleep(10 * time.Millisecond)
+		close(stopCh)
+	}()
+	handleTerminatingSignalDispatch(stopCh, 500*time.Millisecond)
+	if terminatingSignalReceived.Load() {
+		t.Fatal("expected NO fallback flag when SCM path completes cleanly")
+	}
+}
+
+// When requestSCMStop returns true but close(stop) never fires within the
+// timeout, dispatch must set the fallback flag.
+func TestDispatch_SCMAcceptedButTimeout_SetsFallbackFlag(t *testing.T) {
+	withRequestSCMStopFn(t, func() bool { return true })
+	stopCh := make(chan struct{}) // never closed
+	handleTerminatingSignalDispatch(stopCh, 20*time.Millisecond)
+	if !terminatingSignalReceived.Load() {
+		t.Fatal("expected fallback flag set after SCM path timeout")
+	}
+}

cmd/amazon-cloudwatch-agent/shutdown_signal_windows.go

@@ -0,0 +1,139 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+//go:build windows
+
+package main
+
+import (
+	"log"
+	"os"
+
+	winsvc "golang.org/x/sys/windows/svc"
+	winmgr "golang.org/x/sys/windows/svc/mgr"
+)
+
+// Workaround for Windows OS-shutdown deadlock (Event 6008 / Kernel-Power 41):
+// at OS shutdown, csrss delivers CTRL_SHUTDOWN (=> SIGTERM) to the collector
+// but SERVICE_CONTROL_SHUTDOWN goes to the launcher, not us. kardianos'
+// windowsService.Run only watches the SCM channel, so svc.Run stays blocked.
+// requestSCMStop finds this process's SCM entry by matching ProcessId (since
+// kardianos' config Name -- "telegraf" by default -- is not in SCM) and
+// issues STOP so kardianos' normal path can drive svc.Run to return.
+
+// scmManager / scmService: minimal interfaces so tests can substitute fakes.
+type scmManager interface {
+	Disconnect() error
+	ListServices() ([]string, error)
+	OpenService(name string) (scmService, error)
+}
+type scmService interface {
+	Close() error
+	Query() (winsvc.Status, error)
+	Control(cmd winsvc.Cmd) (winsvc.Status, error)
+}
+
+type realSCMManager struct{ m *winmgr.Mgr }
+
+func (r *realSCMManager) Disconnect() error               { return r.m.Disconnect() }
+func (r *realSCMManager) ListServices() ([]string, error) { return r.m.ListServices() }
+func (r *realSCMManager) OpenService(name string) (scmService, error) {
+	s, err := r.m.OpenService(name)
+	if err != nil {
+		return nil, err
+	}
+	return &realSCMService{s: s}, nil
+}
+
+type realSCMService struct{ s *winmgr.Service }
+
+func (r *realSCMService) Close() error                                  { return r.s.Close() }
+func (r *realSCMService) Query() (winsvc.Status, error)                 { return r.s.Query() }
+func (r *realSCMService) Control(cmd winsvc.Cmd) (winsvc.Status, error) { return r.s.Control(cmd) }
+
+// Test seams (single-writer via t.Cleanup, single-reader in shutdown path).
+var (
+	exitFunc       = os.Exit
+	isWinService   = func() bool { return windowsRunAsService() }
+	ownProcessID   = func() uint32 { return uint32(os.Getpid()) }
+	scmConnectFunc = func() (scmManager, error) {
+		m, err := winmgr.Connect()
+		if err != nil {
+			return nil, err
+		}
+		return &realSCMManager{m: m}, nil
+	}
+)
+
+// findOwnSCMServiceName returns the SCM service whose ProcessId matches
+// this process. Empty string if not found. Entries that cannot be opened
+// or queried are skipped.
+func findOwnSCMServiceName(m scmManager) (string, error) {
+	names, err := m.ListServices()
+	if err != nil {
+		return "", err
+	}
+	myPID := ownProcessID()
+	for _, name := range names {
+		s, err := m.OpenService(name)
+		if err != nil {
+			continue
+		}
+		status, err := s.Query()
+		s.Close()
+		if err == nil && status.ProcessId == myPID {
+			return name, nil
+		}
+	}
+	return "", nil
+}
+
+// requestSCMStop issues SERVICE_CONTROL_STOP against this process's own
+// SCM entry. Returns true on success.
+func requestSCMStop() bool {
+	if !isWinService() {
+		return false
+	}
+	m, err := scmConnectFunc()
+	if err != nil {
+		log.Printf("W! Windows service: SCM Connect failed: %v", err)
+		return false
+	}
+	defer m.Disconnect()
+
+	scmName, err := findOwnSCMServiceName(m)
+	if err != nil {
+		log.Printf("W! Windows service: SCM ListServices failed: %v", err)
+		return false
+	}
+	if scmName == "" {
+		log.Printf("W! Windows service: no SCM service found for own PID=%d", ownProcessID())
+		return false
+	}
+
+	s, err := m.OpenService(scmName)
+	if err != nil {
+		log.Printf("W! Windows service: SCM OpenService(%q) failed: %v", scmName, err)
+		return false
+	}
+	defer s.Close()
+	if _, err := s.Control(winsvc.Stop); err != nil {
+		log.Printf("W! Windows service: SCM Control(Stop) on %q failed: %v", scmName, err)
+		return false
+	}
+	log.Printf("I! Windows service: SCM Control(Stop) accepted on %q; waiting for prg.Stop", scmName)
+	return true
+}
+
+// handleTerminatingSignal is the fallback: exit the process iff the flag is
+// set (SCM path unavailable) and we're running as a Windows service.
+func handleTerminatingSignal() {
+	if !terminatingSignalReceived.Load() {
+		return
+	}
+	if !isWinService() {
+		return
+	}
+	log.Println("I! Windows service: SCM stop path unavailable; exiting to release svc.Run")
+	exitFunc(0)
+}