Rework startup of container processes

.github/workflows/main.yml

@@ -472,9 +472,7 @@ jobs:
         run: |
           mkdir ${LOGS_DIR}
           kubectl logs $(kubectl get pods -l component=server -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_server.log
-          kubectl logs $(kubectl get pods -l component=worker-import -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_import.log
-          kubectl logs $(kubectl get pods -l component=worker-export -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_export.log
-          kubectl logs $(kubectl get pods -l component=worker-webhooks -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_webhooks.log
+          kubectl logs $(kubectl get pods -l component=worker-utils -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_workers.log
           kubectl logs $(kubectl get pods -l app.kubernetes.io/name=traefik -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/traefik.log
 
       - name: Uploading "cvat" container logs as an artifact

Dockerfile

@@ -146,7 +146,7 @@
 
 # Add a non-root user
 ENV USER=${USER}
 ENV HOME /home/${USER}
 RUN adduser --shell /bin/bash --disabled-password --gecos "" ${USER}
 
 ARG CLAM_AV="no"
@@ -193,10 +193,10 @@
 # Install and initialize CVAT, copy all necessary files
 COPY cvat/nginx.conf /etc/nginx/nginx.conf
 COPY --chown=${USER} supervisord/ ${HOME}/supervisord
-COPY --chown=${USER} manage.py backend_entrypoint.sh wait_for_deps.sh ${HOME}/
+COPY --chown=${USER} backend_entrypoint.d/ ${HOME}/backend_entrypoint.d
+COPY --chown=${USER} manage.py rqscheduler.py backend_entrypoint.sh wait_for_deps.sh ${HOME}/
 COPY --chown=${USER} utils/ ${HOME}/utils
 COPY --chown=${USER} cvat/ ${HOME}/cvat
-COPY --chown=${USER} rqscheduler.py ${HOME}
 
 ARG COVERAGE_PROCESS_START
 RUN if [ "${COVERAGE_PROCESS_START}" ]; then \

backend_entrypoint.d/cvat.conf

@@ -0,0 +1,12 @@
+(
+    ["annotation"]="smokescreen"
+    ["chunks"]="smokescreen"
+    ["consensus"]=""
+    ["export"]="smokescreen"
+    ["import"]="smokescreen clamav"
+    ["quality_reports"]=""
+    ["cleaning"]="rqscheduler"
+    ["webhooks"]="smokescreen"
+    ["notifications"]=""
+    ["server"]="smokescreen clamav"
+)

backend_entrypoint.sh

@@ -28,12 +28,45 @@ cmd_init() {
     ~/manage.py syncperiodicjobs
 }
 
+_get_includes() {
+    declare -A merged_config
+    for config in ~/backend_entrypoint.d/*.conf; do
+        declare -A config=$(cat $config)
+        for key in "${!config[@]}"; do
+            if [ -v merged_config[$key] ]; then
+                fail "Duplicated component definition: $key"
+            fi
+            merged_config[$key]=${config[$key]}
+        done
+    done
+
+    extra_configs=()
+    for component in "$@"; do
+        if ! [ -v merged_config[$component] ]; then
+            fail "Unexpected worker: $component"
+        fi
+
+        for include in ${merged_config["$component"]}; do
+            if ! [[ ${extra_configs[@]} =~ $include ]] && \
+                ( ! [[ "$include" == "clamav" ]] || [[ "${CLAM_AV:-}" == "yes" ]] ); then
+                extra_configs+=("$include")
+            fi
+        done
+    done
+
+    if [ ${#extra_configs[@]} -gt 0 ]; then
+        printf 'reusable/%s.conf ' "${extra_configs[@]}"
+    fi
+}
+
 cmd_run() {
-    if [ "$#" -ne 1 ]; then
-        fail "run: expected 1 argument"
+    if [ "$#" -eq 0 ]; then
+        fail "run: at least 1 argument is expected"
     fi
 
-    if [ "$1" = "server" ]; then
+    component="$1"
+
+    if [ "$component" = "server" ]; then
         ~/manage.py collectstatic --no-input
     fi
 
@@ -50,7 +83,29 @@ cmd_run() {
         sleep 10
     done
 
-    exec supervisord -c "supervisord/$1.conf"
+    supervisord_includes=""
+    postgres_app_name="cvat:$component"
+    if [ "$component" = "server" ]; then
+        supervisord_includes=$(_get_includes "$component")
+    elif [ "$component" = "worker"  ]; then
+        if [ "$#" -eq 1 ]; then
+            fail "run worker: expected at least 1 queue name"
+        fi
+
+        queue_list="${@:2}"
+        echo "Workers to run: $queue_list"
+        export CVAT_QUEUES=$queue_list
+
+        postgres_app_name+=":${queue_list// /+}"
+
+        supervisord_includes=$(_get_includes $queue_list)
+    fi
+    echo "Additional supervisor configs that will be included: $supervisord_includes"
+
+    export CVAT_POSTGRES_APPLICATION_NAME=$postgres_app_name
+    export CVAT_SUPERVISORD_INCLUDES=$supervisord_includes
+
+    exec supervisord -c "supervisord/$component.conf"
 }
 
 if [ $# -eq 0 ]; then
@@ -59,7 +114,8 @@ if [ $# -eq 0 ]; then
     echo >&2 "available subcommands:"
     echo >&2 "    bash <bash args...>"
     echo >&2 "    init"
-    echo >&2 "    run <config name>"
+    echo >&2 "    run server"
+    echo >&2 "    run worker <list of queues>"
     exit 1
 fi
 

docker-compose.yml

@@ -115,16 +115,16 @@ services:
         aliases:
           - cvat-server
 
-  cvat_utils:
-    container_name: cvat_utils
+  cvat_worker_utils:
+    container_name: cvat_worker_utils
     image: cvat/server:${CVAT_VERSION:-dev}
     restart: always
     depends_on: *backend-deps
     environment:
       <<: *backend-env
       CVAT_REDIS_INMEM_PASSWORD: ''
       NUMPROCS: 1
-    command: run utils
+    command: run worker notifications cleaning
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -140,7 +140,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 2
-    command: run worker.import
+    command: run worker import
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -156,7 +156,7 @@ services:
     environment:
       <<: [*backend-env, *clickhouse-env]
       NUMPROCS: 2
-    command: run worker.export
+    command: run worker export
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -172,7 +172,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 1
-    command: run worker.annotation
+    command: run worker annotation
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -188,7 +188,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 1
-    command: run worker.webhooks
+    command: run worker webhooks
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -204,7 +204,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 1
-    command: run worker.quality_reports
+    command: run worker quality_reports
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -220,7 +220,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 2
-    command: run worker.chunks
+    command: run worker chunks
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys
@@ -236,7 +236,7 @@ services:
     environment:
       <<: *backend-env
       NUMPROCS: 1
-    command: run worker.consensus
+    command: run worker consensus
     volumes:
       - cvat_data:/home/django/data
       - cvat_keys:/home/django/keys

helm-chart/Chart.yaml

@@ -16,7 +16,7 @@ type: application
 # This is the chart version. This version number should be incremented each time you make changes
 # to the chart and its templates, including the app version.
 # Versions are expected to follow Semantic Versioning (https://semver.org/)
-version: 0.14.5
+version: 0.15.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application. Versions are not expected to

helm-chart/templates/_helpers.tpl

@@ -126,27 +126,6 @@ The name of the service account to use for backend pods
   value: "{{ .Release.Name }}-vector"
 - name: DJANGO_LOG_SERVER_PORT
   value: "80"
-{{- end }}
-
-- name: SMOKESCREEN_OPTS
-  value: {{ .Values.smokescreen.opts | toJson }}
-{{- if .Values.nuclio.enabled }}
-- name: CVAT_SERVERLESS
-  value: "1"
-- name: CVAT_NUCLIO_HOST
-  value: "{{ .Release.Name }}-nuclio-dashboard"
-- name: CVAT_NUCLIO_FUNCTION_NAMESPACE
-  value: "{{ .Release.Namespace }}"
-{{- end }}
-
-{{- range $envName, $envValueTemplate := .Values.cvat.backend.extensionEnv }}
-- name: {{ $envName | toYaml }}
-  value: {{ tpl $envValueTemplate $ | toYaml }}
-{{- end }}
-{{- end }}
-
-{{- define "cvat.sharedClickhouseEnv" }}
-{{- if .Values.analytics.enabled }}
 - name: CLICKHOUSE_HOST
   valueFrom:
     secretKeyRef:
@@ -173,6 +152,22 @@ The name of the service account to use for backend pods
       name: cvat-analytics-secret
       key: CLICKHOUSE_PASSWORD
 {{- end }}
+
+- name: SMOKESCREEN_OPTS
+  value: {{ .Values.smokescreen.opts | toJson }}
+{{- if .Values.nuclio.enabled }}
+- name: CVAT_SERVERLESS
+  value: "1"
+- name: CVAT_NUCLIO_HOST
+  value: "{{ .Release.Name }}-nuclio-dashboard"
+- name: CVAT_NUCLIO_FUNCTION_NAMESPACE
+  value: "{{ .Release.Namespace }}"
+{{- end }}
+
+{{- range $envName, $envValueTemplate := .Values.cvat.backend.extensionEnv }}
+- name: {{ $envName | toYaml }}
+  value: {{ tpl $envValueTemplate $ | toYaml }}
+{{- end }}
 {{- end }}
 
 {{- define "cvat.backend.worker.livenessProbe" -}}

helm-chart/templates/cvat_backend/server/deployment.yml

@@ -54,12 +54,13 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "server"]
+          args:
+          - "run"
+          - "server"
           env:
           - name: ALLOWED_HOSTS
             value: {{ $localValues.envs.ALLOWED_HOSTS | squote}}
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
-          {{ include "cvat.sharedClickhouseEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}
           {{- toYaml . | nindent 10 }}
           {{- end }}

helm-chart/templates/cvat_backend/worker_annotation/deployment.yml

@@ -54,7 +54,10 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.annotation"]
+          args:
+          - "run"
+          - "worker"
+          - "annotation"
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/worker_chunks/deployment.yml

@@ -54,7 +54,10 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.chunks"]
+          args:
+          - "run"
+          - "worker"
+          - "chunks"
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/worker_consensus/deployment.yml

@@ -54,7 +54,10 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.consensus"]
+          args:
+          - "run"
+          - "worker"
+          - "consensus"
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/worker_export/deployment.yml

@@ -54,10 +54,12 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.export"]
+          args:
+          - "run"
+          - "worker"
+          - "export"
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
-          {{ include "cvat.sharedClickhouseEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}
           {{- toYaml . | nindent 10 }}
           {{- end }}

helm-chart/templates/cvat_backend/worker_import/deployment.yml

@@ -54,7 +54,10 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.import"]
+          args:
+          - "run"
+          - "worker"
+          - "import"
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}

helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml

@@ -54,7 +54,10 @@ spec:
           resources:
           {{- toYaml . | nindent 12 }}
           {{- end }}
-          args: ["run", "worker.quality_reports"]
+          args:
+          - "run"
+          - "worker"
+          - "quality_reports"
           env:
           {{ include "cvat.sharedBackendEnv" . | indent 10 }}
           {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }}