Skip to content

Commit 5104fd9

Browse files
authored
Added readiness and liveness probes (#8488)
1 parent 5f27977 commit 5104fd9

File tree

20 files changed

+194
-6
lines changed

20 files changed

+194
-6
lines changed
Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
### Added
2+
3+
- \[Helm\] Readiness and liveness probes
4+
(<https://github.com/cvat-ai/cvat/pull/8488>)

cvat/apps/health/management/__init__.py

Whitespace-only changes.

cvat/apps/health/management/commands/__init__.py

Whitespace-only changes.
Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
import os
2+
import platform
3+
from datetime import datetime, timedelta
4+
from django.core.management.base import BaseCommand, CommandError
5+
from django.conf import settings
6+
from rq.worker import Worker
7+
import django_rq
8+
9+
10+
class Command(BaseCommand):
11+
help = "Check worker liveness in specified queues"
12+
13+
def add_arguments(self, parser):
14+
parser.add_argument("queue_names", nargs="+", type=str)
15+
16+
def handle(self, *args, **options):
17+
hostname = platform.node()
18+
for queue_name in options["queue_names"]:
19+
if queue_name not in settings.RQ_QUEUES:
20+
raise CommandError(f"Queue {queue_name} is not defined")
21+
22+
connection = django_rq.get_connection(queue_name)
23+
workers = [w for w in Worker.all(connection) if queue_name in w.queue_names() and w.hostname == hostname]
24+
25+
expected_workers = int(os.getenv("NUMPROCS", 1))
26+
27+
if len(workers) != expected_workers:
28+
raise CommandError("Number of registered workers does not match the expected number, " \
29+
f"actual: {len(workers)}, expected: {expected_workers}")
30+
for worker in workers:
31+
if datetime.now() - worker.last_heartbeat > timedelta(seconds=worker.worker_ttl):
32+
raise CommandError(f"It seems that worker {worker.name}, pid: {worker.pid} is dead")

helm-chart/Chart.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ type: application
1616
# This is the chart version. This version number should be incremented each time you make changes
1717
# to the chart and its templates, including the app version.
1818
# Versions are expected to follow Semantic Versioning (https://semver.org/)
19-
version: 0.13.2
19+
version: 0.14.0
2020

2121
# This is the version number of the application being deployed. This version number should be
2222
# incremented each time you make changes to the application. Versions are not expected to

helm-chart/templates/_helpers.tpl

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -169,3 +169,18 @@ The name of the service account to use for backend pods
169169
key: CLICKHOUSE_PASSWORD
170170
{{- end }}
171171
{{- end }}
172+
173+
{{- define "cvat.backend.worker.livenessProbe" -}}
174+
{{- if .livenessProbe.enabled }}
175+
livenessProbe:
176+
exec:
177+
command:
178+
- python
179+
- manage.py
180+
- workerprobe
181+
{{- range .args }}
182+
- {{ . }}
183+
{{- end }}
184+
{{ toYaml (omit .livenessProbe "enabled") | indent 2}}
185+
{{- end }}
186+
{{- end }}

helm-chart/templates/cvat_backend/server/deployment.yml

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,20 @@ spec:
6565
{{- end }}
6666
ports:
6767
- containerPort: 8080
68+
{{- if $localValues.readinessProbe.enabled }}
69+
readinessProbe:
70+
httpGet:
71+
path: /api/server/about
72+
port: 8080
73+
{{- toYaml (omit $localValues.readinessProbe "enabled") | nindent 12 }}
74+
{{- end }}
75+
{{- if $localValues.livenessProbe.enabled }}
76+
livenessProbe:
77+
httpGet:
78+
path: /api/server/about
79+
port: 8080
80+
{{- toYaml (omit $localValues.livenessProbe "enabled") | nindent 12 }}
81+
{{- end }}
6882
volumeMounts:
6983
{{- if not .Values.cvat.backend.disableDistinctCachePerService }}
7084
- mountPath: /home/django/data/cache

helm-chart/templates/cvat_backend/utils/deployment.yml

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,8 +60,9 @@ spec:
6060
{{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}
6161
{{- toYaml . | nindent 10 }}
6262
{{- end }}
63-
ports:
64-
- containerPort: 8080
63+
{{- $probeArgs := list "notifications" "cleaning" -}}
64+
{{- $probeConfig := dict "args" $probeArgs "livenessProbe" $.Values.cvat.backend.worker.livenessProbe -}}
65+
{{ include "cvat.backend.worker.livenessProbe" $probeConfig | indent 10 }}
6566
volumeMounts:
6667
{{- if not .Values.cvat.backend.disableDistinctCachePerService }}
6768
- mountPath: /home/django/data/cache

helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,9 @@ spec:
6161
{{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}
6262
{{- toYaml . | nindent 10 }}
6363
{{- end }}
64+
{{- $probeArgs := list "analytics_reports" -}}
65+
{{- $probeConfig := dict "args" $probeArgs "livenessProbe" $.Values.cvat.backend.worker.livenessProbe -}}
66+
{{ include "cvat.backend.worker.livenessProbe" $probeConfig | indent 10 }}
6467
{{- with concat .Values.cvat.backend.additionalVolumeMounts $localValues.additionalVolumeMounts }}
6568
volumeMounts:
6669
{{- toYaml . | nindent 10 }}

helm-chart/templates/cvat_backend/worker_annotation/deployment.yml

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,9 @@ spec:
6060
{{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}
6161
{{- toYaml . | nindent 10 }}
6262
{{- end }}
63+
{{- $probeArgs := list "annotation" -}}
64+
{{- $probeConfig := dict "args" $probeArgs "livenessProbe" $.Values.cvat.backend.worker.livenessProbe -}}
65+
{{ include "cvat.backend.worker.livenessProbe" $probeConfig | indent 10 }}
6366
volumeMounts:
6467
{{- if not .Values.cvat.backend.disableDistinctCachePerService }}
6568
- mountPath: /home/django/data/cache

0 commit comments

Comments
 (0)