-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathstatus.py
More file actions
77 lines (64 loc) · 2.56 KB
/
Copy pathstatus.py
File metadata and controls
77 lines (64 loc) · 2.56 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
from typing import Literal
from litestar import Controller, get, Request
from litestar.response import Response
from pydantic import BaseModel
from celery.result import AsyncResult
from geoapi.tasks.health import check_worker
from geoapi.log import logging
from geoapi.utils.decorators import not_anonymous_guard
logger = logging.getLogger(__name__)
WORKER_CHECK_TIMEOUT = (
40 # seconds; request timeout is 60s, leaving buffer for response
)
class StatusResponse(BaseModel):
status: str
class ComponentStatus(BaseModel):
status: Literal["ok", "error"]
detail: str | None = None
class WorkerStatusResponse(BaseModel):
overall: Literal["ok", "error"]
components: dict[str, ComponentStatus]
class StatusController(Controller):
path = "/status"
@get("/", tags=["status"])
async def get_status(self, request: Request) -> StatusResponse:
"""Unauthenticated liveness check for load balancer."""
return StatusResponse(status="OK")
@get("/complete", tags=["status"], guards=[not_anonymous_guard])
async def get_status_complete(
self, request: Request
) -> Response[WorkerStatusResponse]:
"""
Authenticated health check. Submits a Celery task and waits for
the result, validating worker + Redis connectivity from within the
worker network.
"""
# Submit at highest priority so health checks aren't blocked by queued default tasks
task = check_worker.apply_async(priority=10)
try:
result = AsyncResult(task.id).get(timeout=WORKER_CHECK_TIMEOUT)
except Exception as e:
logger.error(
f"Worker health check timed out or failed for user:{request.user.username}: {e}"
)
body = WorkerStatusResponse(
overall="error",
components={
"worker": ComponentStatus(
status="error", detail="Worker unavailable or timed out"
)
},
)
return Response(content=body, status_code=503)
body = WorkerStatusResponse(
overall=result["overall"],
components={
k: ComponentStatus(**v) for k, v in result["components"].items()
},
)
status_code = 200 if result["overall"] == "ok" else 503
if status_code != 200:
logger.warning(
f"Worker health check degraded/failed for user:{request.user.username} result:{result}"
)
return Response(content=body, status_code=status_code)