Rework startup of container processes

.github/workflows/main.yml

@@ -474,9 +474,7 @@ jobs:
         run: |
           mkdir ${LOGS_DIR}
           kubectl logs $(kubectl get pods -l component=server -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_server.log
-          kubectl logs $(kubectl get pods -l component=worker-import -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_import.log
-          kubectl logs $(kubectl get pods -l component=worker-export -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_export.log
-          kubectl logs $(kubectl get pods -l component=worker-webhooks -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_webhooks.log
+          kubectl logs $(kubectl get pods -l component=worker-utils -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_workers.log
           kubectl logs $(kubectl get pods -l app.kubernetes.io/name=traefik -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/traefik.log
 
       - name: Uploading "cvat" container logs as an artifact

backend_entrypoint.sh

@@ -28,12 +28,42 @@ cmd_init() {
     ~/manage.py syncperiodicjobs
 }
 
+_get_supervisord_includes() {
+    extra_configs=""
+    for arg in "$@"; do
+        if [[ "$arg" = include=* ]]; then
+            extra_configs+=" ${arg#include=}_reusable.conf"
+        fi
+    done
+    echo $extra_configs
+}
+
+_get_worker_list() {
+    workers=""
+    for arg in "$@"; do
+        if [[ "$arg" != include=* ]]; then
+            workers+=" $arg"
+        fi
+    done
+    echo $workers
+}
+
+_get_app_label() {
+    label=$1
+    for arg in "${@:2}"; do
+        label+="+${arg}"
+    done
+    echo $label
+}
+
 cmd_run() {
-    if [ "$#" -ne 1 ]; then
-        fail "run: expected 1 argument"
+    if [ "$#" -eq 0 ]; then
+        fail "run: at least 1 argument is expected"
     fi
 
-    if [ "$1" = "server" ]; then
+    component="$1"
+
+    if [ "$component" = "server" ]; then
         ~/manage.py collectstatic --no-input
     fi
 
@@ -50,7 +80,24 @@ cmd_run() {
         sleep 10
     done
 
-    exec supervisord -c "supervisord/$1.conf"
+    supervisord_includes=$(_get_supervisord_includes "${@:2}")
+    postgres_app_name="cvat:$component"
+
+    if [ "$component" = "worker"  ]; then
+        if [ "$#" = 1 ]; then
+            fail "run worker: expected at least 1 worker name"
+        fi
+
+        worker_list=$(_get_worker_list "${@:2}")
+        echo "Workers to run: $worker_list"
+        export CVAT_WORKERS=$worker_list
+        postgres_app_name+=":$(_get_app_label $worker_list)"
+    fi
+
+    export CVAT_POSTGRES_APPLICATION_NAME=$postgres_app_name
+    export CVAT_SUPERVISORD_INCLUDES=$supervisord_includes
+
+    exec supervisord -c "supervisord/$component.conf"
 }
 
 if [ $# -eq 0 ]; then
@@ -59,7 +106,8 @@ if [ $# -eq 0 ]; then
     echo >&2 "available subcommands:"
     echo >&2 "    bash <bash args...>"
     echo >&2 "    init"
-    echo >&2 "    run <config name>"
+    echo >&2 "    run server <list of optional services in the format include=config, e.g. include=smokescreen>"
+    echo >&2 "    run worker <list of workers> <list of optional services in the format include=config, e.g. include=smokescreen>"
     exit 1
 fi
 

docker-compose.external_db.yml

@@ -20,7 +20,7 @@ services:
       replicas: 0
 
   cvat_server: *backend-settings
-  cvat_utils: *backend-settings
+  cvat_worker_utils: *backend-settings
   cvat_worker_analytics_reports: *backend-settings
   cvat_worker_annotation: *backend-settings
   cvat_worker_export: *backend-settings

docker-compose.yml

@@ -97,7 +97,7 @@ services:
       CVAT_ANALYTICS: 1
       CVAT_BASE_URL:
       ONE_RUNNING_JOB_IN_QUEUE_PER_USER:
-    command: init run server
+    command: init run server include=clamav include=smokescreen
     labels:
       traefik.enable: "true"
       traefik.http.services.cvat.loadbalancer.server.port: "8080"
@@ -114,16 +114,16 @@ services:
         aliases:
           - cvat-server
 
-  cvat_utils:
-    container_name: cvat_utils
+  cvat_worker_utils:
+    container_name: cvat_worker_utils
     image: cvat/server:${CVAT_VERSION:-dev}
     restart: always
     depends_on: *backend-deps
     environment:
       <<: *backend-env
       CVAT_REDIS_INMEM_PASSWORD: ''
       NUMPROCS: 1
-    command: run utils
+    command: run worker notifications cleaning include=rqscheduler
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -139,7 +139,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 2
-    command: run worker.import
+    command: run worker import include=smokescreen include=clamav
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -155,7 +155,7 @@ services:
     environment:
       <<: [*backend-env, *clickhouse-env]
       NUMPROCS: 2
-    command: run worker.export
+    command: run worker export include=smokescreen
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -171,7 +171,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 1
-    command: run worker.annotation
+    command: run worker annotation include=smokescreen
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -187,7 +187,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 1
-    command: run worker.webhooks
+    command: run worker webhooks include=smokescreen
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -203,7 +203,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 1
-    command: run worker.quality_reports
+    command: run worker quality_reports
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -219,7 +219,7 @@ services:
     environment:
       <<: [*backend-env, *clickhouse-env]
       NUMPROCS: 2
-    command: run worker.analytics_reports
+    command: run worker analytics_reports
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -235,7 +235,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 2
-    command: run worker.chunks
+    command: run worker chunks include=smokescreen
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -251,7 +251,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 1
-    command: run worker.consensus
+    command: run worker consensus
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys

helm-chart/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.14.4
+version: 0.15.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm-chart/templates/cvat_backend/server/deployment.yml

@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "server"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           - name: ALLOWED_HOSTS
             value: {{ $localValues.envs.ALLOWED_HOSTS | squote}}

helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml

@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.analytics_reports"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{ include "cvat.sharedClickhouseEnv" . | indent 10 }}

helm-chart/templates/cvat_backend/worker_annotation/deployment.yml

@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.annotation"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/worker_chunks/deployment.yml

@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.chunks"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/worker_consensus/deployment.yml

@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.consensus"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/worker_export/deployment.yml

@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.export"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{ include "cvat.sharedClickhouseEnv" . | indent 10 }}

helm-chart/templates/cvat_backend/worker_import/deployment.yml

@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.import"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml

@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.quality_reports"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/utils/deployment.yml → helm-chart/templates/cvat_backend/worker_utils/deployment.yml

@@ -1,14 +1,14 @@
-{{- $localValues := .Values.cvat.backend.utils -}}
+{{- $localValues := .Values.cvat.backend.worker.utils -}}
 
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ .Release.Name }}-backend-utils
+  name: {{ .Release.Name }}-backend-worker-utils
   namespace: {{ .Release.Namespace }}
   labels:
     app: cvat-app
     tier: backend
-    component: utils
+    component: worker-utils
     {{- include "cvat.labels" . | nindent 4 }}
     {{- with merge $localValues.labels .Values.cvat.backend.labels }}
     {{- toYaml . | nindent 4 }}
@@ -29,13 +29,13 @@ spec:
       {{- end }}
       app: cvat-app
       tier: backend
-      component: utils
+      component: worker-utils
   template:
     metadata:
       labels:
         app: cvat-app
         tier: backend
-        component: utils
+        component: worker-utils
         {{- include "cvat.labels" . | nindent 8 }}
         {{- with merge $localValues.labels .Values.cvat.backend.labels }}
         {{- toYaml . | nindent 8 }}
@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "utils"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/worker_webhooks/deployment.yml

@@ -54,7 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.webhooks"]
+          args:
+          {{- with $localValues.args }}
+          {{- range . }}
+          - {{ . | quote }}
+          {{- end }}
+          {{- end }}
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}