Require sustained, restart-free checks in wait

Settle only after stableChecks consecutive checks with full replicas, no
active update, and no newly failed tasks. A new task failure keeps the
service unsettled for a full stableChecks window, so a container that
crash-loops faster than that window can never report success.

Author: firecow

src/commands/wait-cmd.ts

@@ -26,12 +26,18 @@ export async function handler (args: ArgumentsCamelCase) {
     assertNumber(timeout, "timeout must be a number in ms");
     const interval = args.interval;
     assertNumber(interval, "interval must be a number in ms");
+    const stableChecks = args.stableChecks;
+    assertNumber(stableChecks, "stableChecks must be a number");
 
     const dockerode = new Docker();
 
     console.log(`Awaiting task reconciliation for ${timeout}ms`);
 
     let services: Service[], tasks: Task[], timedout, bail, serviceStateMap;
+    let stableStreak = 0;
+    const seenFailedTaskIds = new Set<string>();
+    const restartCooldown = new Map<string, number>();
+    let firstCheck = true;
     const start = Date.now();
     do {
         // To prevent high cpu usage
@@ -55,17 +61,40 @@ export async function handler (args: ArgumentsCamelCase) {
             }
         }
 
+        for (const t of tasks) {
+            if (!["failed", "rejected"].includes(t.Status.State) || seenFailedTaskIds.has(t.ID)) {
+                continue;
+            }
+            seenFailedTaskIds.add(t.ID);
+            if (!firstCheck) {
+                restartCooldown.set(t.ServiceID, stableChecks);
+            }
+        }
+        firstCheck = false;
+
+        for (const [serviceId, remaining] of restartCooldown) {
+            serviceStateMap.set(serviceId, "restarting");
+            restartCooldown.set(serviceId, remaining - 1);
+            if (remaining - 1 <= 0) {
+                restartCooldown.delete(serviceId);
+            }
+        }
+
         const servicesUpdating = [...serviceStateMap.entries()];
 
-        bail = servicesUpdating.length === 0;
-        if (!bail) {
+        if (servicesUpdating.length === 0) {
+            stableStreak++;
+        } else {
+            stableStreak = 0;
             for (const [serviceId, state] of servicesUpdating) {
                 const serviceName = services.find((s) => s.ID === serviceId)?.Spec?.Name;
                 assert(serviceName != null, "serviceName must be a string");
                 const errMsg = tasks.find((t) => t.ServiceID === serviceId && t.Status.Err)?.Status.Err;
                 console.log(`${serviceName} is in ${state} state${errMsg ? ", error: '" + errMsg + "'" : ""}`);
             }
         }
+
+        bail = stableStreak >= stableChecks;
     } while (!timedout && !bail);
 
     if (timedout) {
@@ -88,6 +117,11 @@ export function builder (yargs: Argv) {
         description: "How often reconciliation should run",
         default: 5000,
     });
+    yargs.positional("stableChecks", {
+        type: "number",
+        description: "Consecutive settled checks required before success",
+        default: 3,
+    });
     yargs.hide("help");
     yargs.hide("version");
     return yargs;