Skip to content

Commit bd8807a

Browse files
committed
feat: add service readiness check after managed update and enhance failure detail logging
1 parent 143a887 commit bd8807a

1 file changed

Lines changed: 73 additions & 0 deletions

File tree

internal/agentctl/update.go

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ const (
2424
releaseManifestAssetName = "release-manifest.json"
2525
updateProgressRequestTimeout = 10 * time.Second
2626
updateDownloadTimeout = 10 * time.Minute
27+
updateServiceReadyTimeout = 30 * time.Second
2728
)
2829

2930
type updateOptions struct {
@@ -286,6 +287,10 @@ func (c CLI) applyManagedUpdate(
286287
)
287288
}
288289

290+
if err := waitForRestartedService(ctx, spec.ServiceName); err != nil {
291+
return err
292+
}
293+
289294
report(
290295
"waiting_for_reconnect",
291296
95,
@@ -567,3 +572,71 @@ func sanitizeSystemdUnitComponent(value string) string {
567572
}
568573
return cleaned
569574
}
575+
576+
func waitForRestartedService(ctx context.Context, serviceName string) error {
577+
deadline := time.Now().Add(updateServiceReadyTimeout)
578+
for {
579+
if err := ctx.Err(); err != nil {
580+
return err
581+
}
582+
583+
checkCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
584+
cmd := exec.CommandContext(
585+
checkCtx,
586+
"systemctl",
587+
"is-active",
588+
"--quiet",
589+
serviceName,
590+
)
591+
err := cmd.Run()
592+
cancel()
593+
if err == nil {
594+
return nil
595+
}
596+
597+
if time.Now().After(deadline) {
598+
return fmt.Errorf(
599+
"service %s did not become active after restart: %s",
600+
serviceName,
601+
readServiceFailureDetail(ctx, serviceName),
602+
)
603+
}
604+
605+
time.Sleep(1 * time.Second)
606+
}
607+
}
608+
609+
func readServiceFailureDetail(ctx context.Context, serviceName string) string {
610+
statusCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
611+
defer cancel()
612+
613+
cmd := exec.CommandContext(
614+
statusCtx,
615+
"journalctl",
616+
"-u",
617+
serviceName,
618+
"-n",
619+
"25",
620+
"--no-pager",
621+
)
622+
output, err := cmd.CombinedOutput()
623+
if err != nil {
624+
message := strings.TrimSpace(string(output))
625+
if message == "" {
626+
return fmt.Sprintf("unable to read journal: %v", err)
627+
}
628+
return fmt.Sprintf("unable to read journal: %v: %s", err, message)
629+
}
630+
631+
lines := strings.Split(strings.TrimSpace(string(output)), "\n")
632+
if len(lines) == 0 || strings.TrimSpace(lines[len(lines)-1]) == "" {
633+
return "journal did not return any recent entries"
634+
}
635+
636+
const maxLines = 5
637+
if len(lines) > maxLines {
638+
lines = lines[len(lines)-maxLines:]
639+
}
640+
641+
return strings.Join(lines, " | ")
642+
}

0 commit comments

Comments
 (0)