Added readiness and liveness probes (#8488)

Author: azhavoro

changelog.d/20241009_164505_andrey_k8s_probes.md

@@ -0,0 +1,4 @@
+### Added
+
+- \[Helm\] Readiness and liveness probes
+  (<https://github.com/cvat-ai/cvat/pull/8488>)

cvat/apps/health/management/__init__.py

@@ -0,0 +1,32 @@
+import os
+import platform
+from datetime import datetime, timedelta
+from django.core.management.base import BaseCommand, CommandError
+from django.conf import settings
+from rq.worker import Worker
+import django_rq
+
+
+class Command(BaseCommand):
+    help = "Check worker liveness in specified queues"
+
+    def add_arguments(self, parser):
+        parser.add_argument("queue_names", nargs="+", type=str)
+
+    def handle(self, *args, **options):
+        hostname = platform.node()
+        for queue_name in options["queue_names"]:
+            if queue_name not in settings.RQ_QUEUES:
+                raise CommandError(f"Queue {queue_name} is not defined")
+
+            connection = django_rq.get_connection(queue_name)
+            workers = [w for w in Worker.all(connection) if queue_name in w.queue_names() and w.hostname == hostname]
+
+            expected_workers = int(os.getenv("NUMPROCS", 1))
+
+            if len(workers) != expected_workers:
+                raise CommandError("Number of registered workers does not match the expected number, " \
+                                    f"actual: {len(workers)}, expected: {expected_workers}")
+            for worker in workers:
+                if datetime.now() - worker.last_heartbeat > timedelta(seconds=worker.worker_ttl):
+                    raise CommandError(f"It seems that worker {worker.name}, pid: {worker.pid} is dead")

cvat/apps/health/management/commands/__init__.py

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.13.2
+version: 0.14.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

cvat/apps/health/management/commands/workerprobe.py

@@ -169,3 +169,18 @@ The name of the service account to use for backend pods
       key: CLICKHOUSE_PASSWORD
 {{- end }}
 {{- end }}
+
+{{- define "cvat.backend.worker.livenessProbe" -}}
+{{- if .livenessProbe.enabled }}
+livenessProbe:
+  exec:
+    command:
+    - python
+    - manage.py
+    - workerprobe
+    {{- range .args }}
+    - {{ . }}
+    {{- end }}
+{{ toYaml (omit .livenessProbe "enabled") | indent 2}}
+{{- end }}
+{{- end }}

helm-chart/Chart.yaml

@@ -65,6 +65,20 @@ spec:
           {{- end }}
           ports:
           - containerPort: 8080
+          {{- if $localValues.readinessProbe.enabled }}
+          readinessProbe:
+            httpGet:
+              path: /api/server/about
+              port: 8080
+            {{- toYaml (omit $localValues.readinessProbe "enabled") | nindent 12 }}
+          {{- end }}
+          {{- if $localValues.livenessProbe.enabled }}
+          livenessProbe:
+            httpGet:
+              path: /api/server/about
+              port: 8080
+            {{- toYaml (omit $localValues.livenessProbe "enabled") | nindent 12 }}
+          {{- end }}
           volumeMounts:
           {{- if not .Values.cvat.backend.disableDistinctCachePerService }}
           - mountPath: /home/django/data/cache

helm-chart/templates/_helpers.tpl

@@ -60,8 +60,9 @@ spec:
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}
           {{- toYaml . | nindent 10 }}
           {{- end }}
-          ports:
-          - containerPort: 8080
+          {{- $probeArgs := list "notifications" "cleaning" -}}
+          {{- $probeConfig := dict "args" $probeArgs "livenessProbe" $.Values.cvat.backend.worker.livenessProbe -}}
+          {{ include "cvat.backend.worker.livenessProbe" $probeConfig | indent 10 }}
           volumeMounts:
           {{- if not .Values.cvat.backend.disableDistinctCachePerService }}
           - mountPath: /home/django/data/cache

helm-chart/templates/cvat_backend/server/deployment.yml

@@ -61,6 +61,9 @@ spec:
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}
           {{- toYaml . | nindent 10 }}
           {{- end }}
+          {{- $probeArgs := list "analytics_reports" -}}
+          {{- $probeConfig := dict "args" $probeArgs "livenessProbe" $.Values.cvat.backend.worker.livenessProbe -}}
+          {{ include "cvat.backend.worker.livenessProbe" $probeConfig | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalVolumeMounts $localValues.additionalVolumeMounts }}
           volumeMounts:
           {{- toYaml . | nindent 10 }}

helm-chart/templates/cvat_backend/utils/deployment.yml

@@ -60,6 +60,9 @@ spec:
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}
           {{- toYaml . | nindent 10 }}
           {{- end }}
+          {{- $probeArgs := list "annotation" -}}
+          {{- $probeConfig := dict "args" $probeArgs "livenessProbe" $.Values.cvat.backend.worker.livenessProbe -}}
+          {{ include "cvat.backend.worker.livenessProbe" $probeConfig | indent 10 }}
           volumeMounts:
           {{- if not .Values.cvat.backend.disableDistinctCachePerService }}
           - mountPath: /home/django/data/cache