diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 57f014bb2226..3a0c81a34e88 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -472,9 +472,7 @@ jobs: run: | mkdir ${LOGS_DIR} kubectl logs $(kubectl get pods -l component=server -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_server.log - kubectl logs $(kubectl get pods -l component=worker-import -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_import.log - kubectl logs $(kubectl get pods -l component=worker-export -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_export.log - kubectl logs $(kubectl get pods -l component=worker-webhooks -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_webhooks.log + kubectl logs $(kubectl get pods -l component=worker-utils -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_workers.log kubectl logs $(kubectl get pods -l app.kubernetes.io/name=traefik -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/traefik.log - name: Uploading "cvat" container logs as an artifact diff --git a/Dockerfile b/Dockerfile index 28aa4593fafa..f67e22f5ad77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -193,10 +193,10 @@ RUN python -m pip uninstall -y pip # Install and initialize CVAT, copy all necessary files COPY cvat/nginx.conf /etc/nginx/nginx.conf COPY --chown=${USER} supervisord/ ${HOME}/supervisord -COPY --chown=${USER} manage.py backend_entrypoint.sh wait_for_deps.sh ${HOME}/ +COPY --chown=${USER} backend_entrypoint.d/ ${HOME}/backend_entrypoint.d +COPY --chown=${USER} manage.py rqscheduler.py backend_entrypoint.sh wait_for_deps.sh ${HOME}/ COPY --chown=${USER} utils/ ${HOME}/utils COPY --chown=${USER} cvat/ ${HOME}/cvat -COPY --chown=${USER} rqscheduler.py ${HOME} ARG COVERAGE_PROCESS_START RUN if [ "${COVERAGE_PROCESS_START}" ]; then \ diff --git a/backend_entrypoint.d/cvat.conf b/backend_entrypoint.d/cvat.conf new file mode 100644 index 000000000000..f743603def7d --- /dev/null +++ b/backend_entrypoint.d/cvat.conf @@ -0,0 +1,12 @@ +( + ["annotation"]="smokescreen" + ["chunks"]="smokescreen" + ["consensus"]="" + ["export"]="smokescreen" + ["import"]="smokescreen clamav" + ["quality_reports"]="" + ["cleaning"]="rqscheduler" + ["webhooks"]="smokescreen" + ["notifications"]="" + ["server"]="smokescreen clamav" +) diff --git a/backend_entrypoint.sh b/backend_entrypoint.sh index 5b7f24ebe7e4..690a10937997 100755 --- a/backend_entrypoint.sh +++ b/backend_entrypoint.sh @@ -28,12 +28,45 @@ cmd_init() { ~/manage.py syncperiodicjobs } +_get_includes() { + declare -A merged_config + for config in ~/backend_entrypoint.d/*.conf; do + declare -A config=$(cat $config) + for key in "${!config[@]}"; do + if [ -v merged_config[$key] ]; then + fail "Duplicated component definition: $key" + fi + merged_config[$key]=${config[$key]} + done + done + + extra_configs=() + for component in "$@"; do + if ! [ -v merged_config[$component] ]; then + fail "Unexpected worker: $component" + fi + + for include in ${merged_config["$component"]}; do + if ! [[ ${extra_configs[@]} =~ $include ]] && \ + ( ! [[ "$include" == "clamav" ]] || [[ "${CLAM_AV:-}" == "yes" ]] ); then + extra_configs+=("$include") + fi + done + done + + if [ ${#extra_configs[@]} -gt 0 ]; then + printf 'reusable/%s.conf ' "${extra_configs[@]}" + fi +} + cmd_run() { - if [ "$#" -ne 1 ]; then - fail "run: expected 1 argument" + if [ "$#" -eq 0 ]; then + fail "run: at least 1 argument is expected" fi - if [ "$1" = "server" ]; then + component="$1" + + if [ "$component" = "server" ]; then ~/manage.py collectstatic --no-input fi @@ -50,7 +83,29 @@ cmd_run() { sleep 10 done - exec supervisord -c "supervisord/$1.conf" + supervisord_includes="" + postgres_app_name="cvat:$component" + if [ "$component" = "server" ]; then + supervisord_includes=$(_get_includes "$component") + elif [ "$component" = "worker" ]; then + if [ "$#" -eq 1 ]; then + fail "run worker: expected at least 1 queue name" + fi + + queue_list="${@:2}" + echo "Workers to run: $queue_list" + export CVAT_QUEUES=$queue_list + + postgres_app_name+=":${queue_list// /+}" + + supervisord_includes=$(_get_includes $queue_list) + fi + echo "Additional supervisor configs that will be included: $supervisord_includes" + + export CVAT_POSTGRES_APPLICATION_NAME=$postgres_app_name + export CVAT_SUPERVISORD_INCLUDES=$supervisord_includes + + exec supervisord -c "supervisord/$component.conf" } if [ $# -eq 0 ]; then @@ -59,7 +114,8 @@ if [ $# -eq 0 ]; then echo >&2 "available subcommands:" echo >&2 " bash " echo >&2 " init" - echo >&2 " run " + echo >&2 " run server" + echo >&2 " run worker " exit 1 fi diff --git a/docker-compose.yml b/docker-compose.yml index d9edf0965a2d..f8938856c4db 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -115,8 +115,8 @@ services: aliases: - cvat-server - cvat_utils: - container_name: cvat_utils + cvat_worker_utils: + container_name: cvat_worker_utils image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: *backend-deps @@ -124,7 +124,7 @@ services: <<: *backend-env CVAT_REDIS_INMEM_PASSWORD: '' NUMPROCS: 1 - command: run utils + command: run worker notifications cleaning volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -140,7 +140,7 @@ services: environment: <<: *backend-env NUMPROCS: 2 - command: run worker.import + command: run worker import volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -156,7 +156,7 @@ services: environment: <<: [*backend-env, *clickhouse-env] NUMPROCS: 2 - command: run worker.export + command: run worker export volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -172,7 +172,7 @@ services: environment: <<: *backend-env NUMPROCS: 1 - command: run worker.annotation + command: run worker annotation volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -188,7 +188,7 @@ services: environment: <<: *backend-env NUMPROCS: 1 - command: run worker.webhooks + command: run worker webhooks volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -204,7 +204,7 @@ services: environment: <<: *backend-env NUMPROCS: 1 - command: run worker.quality_reports + command: run worker quality_reports volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -220,7 +220,7 @@ services: environment: <<: *backend-env NUMPROCS: 2 - command: run worker.chunks + command: run worker chunks volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -236,7 +236,7 @@ services: environment: <<: *backend-env NUMPROCS: 1 - command: run worker.consensus + command: run worker consensus volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 91fa27399cc3..248d93f9b99e 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.14.5 +version: 0.15.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm-chart/templates/_helpers.tpl b/helm-chart/templates/_helpers.tpl index b27b7aa34666..f73a14afb07c 100644 --- a/helm-chart/templates/_helpers.tpl +++ b/helm-chart/templates/_helpers.tpl @@ -126,27 +126,6 @@ The name of the service account to use for backend pods value: "{{ .Release.Name }}-vector" - name: DJANGO_LOG_SERVER_PORT value: "80" -{{- end }} - -- name: SMOKESCREEN_OPTS - value: {{ .Values.smokescreen.opts | toJson }} -{{- if .Values.nuclio.enabled }} -- name: CVAT_SERVERLESS - value: "1" -- name: CVAT_NUCLIO_HOST - value: "{{ .Release.Name }}-nuclio-dashboard" -- name: CVAT_NUCLIO_FUNCTION_NAMESPACE - value: "{{ .Release.Namespace }}" -{{- end }} - -{{- range $envName, $envValueTemplate := .Values.cvat.backend.extensionEnv }} -- name: {{ $envName | toYaml }} - value: {{ tpl $envValueTemplate $ | toYaml }} -{{- end }} -{{- end }} - -{{- define "cvat.sharedClickhouseEnv" }} -{{- if .Values.analytics.enabled }} - name: CLICKHOUSE_HOST valueFrom: secretKeyRef: @@ -173,6 +152,22 @@ The name of the service account to use for backend pods name: cvat-analytics-secret key: CLICKHOUSE_PASSWORD {{- end }} + +- name: SMOKESCREEN_OPTS + value: {{ .Values.smokescreen.opts | toJson }} +{{- if .Values.nuclio.enabled }} +- name: CVAT_SERVERLESS + value: "1" +- name: CVAT_NUCLIO_HOST + value: "{{ .Release.Name }}-nuclio-dashboard" +- name: CVAT_NUCLIO_FUNCTION_NAMESPACE + value: "{{ .Release.Namespace }}" +{{- end }} + +{{- range $envName, $envValueTemplate := .Values.cvat.backend.extensionEnv }} +- name: {{ $envName | toYaml }} + value: {{ tpl $envValueTemplate $ | toYaml }} +{{- end }} {{- end }} {{- define "cvat.backend.worker.livenessProbe" -}} diff --git a/helm-chart/templates/cvat_backend/server/deployment.yml b/helm-chart/templates/cvat_backend/server/deployment.yml index 28daf0da7cb7..84b7264edd1b 100644 --- a/helm-chart/templates/cvat_backend/server/deployment.yml +++ b/helm-chart/templates/cvat_backend/server/deployment.yml @@ -54,12 +54,13 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "server"] + args: + - "run" + - "server" env: - name: ALLOWED_HOSTS value: {{ $localValues.envs.ALLOWED_HOSTS | squote}} {{ include "cvat.sharedBackendEnv" . | indent 10 }} - {{ include "cvat.sharedClickhouseEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} {{- toYaml . | nindent 10 }} {{- end }} diff --git a/helm-chart/templates/cvat_backend/worker_annotation/deployment.yml b/helm-chart/templates/cvat_backend/worker_annotation/deployment.yml index a2439325ca45..d29b3421f23f 100644 --- a/helm-chart/templates/cvat_backend/worker_annotation/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_annotation/deployment.yml @@ -54,7 +54,10 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.annotation"] + args: + - "run" + - "worker" + - "annotation" env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_chunks/deployment.yml b/helm-chart/templates/cvat_backend/worker_chunks/deployment.yml index 74e80b1b185d..a03e60248945 100644 --- a/helm-chart/templates/cvat_backend/worker_chunks/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_chunks/deployment.yml @@ -54,7 +54,10 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.chunks"] + args: + - "run" + - "worker" + - "chunks" env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml b/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml index c96b5aa2e96a..60ae77f25ec8 100644 --- a/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml @@ -54,7 +54,10 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.consensus"] + args: + - "run" + - "worker" + - "consensus" env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_export/deployment.yml b/helm-chart/templates/cvat_backend/worker_export/deployment.yml index 6cb755dbd01b..1853cc463ba9 100644 --- a/helm-chart/templates/cvat_backend/worker_export/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_export/deployment.yml @@ -54,10 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.export"] + args: + - "run" + - "worker" + - "export" env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} - {{ include "cvat.sharedClickhouseEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} {{- toYaml . | nindent 10 }} {{- end }} diff --git a/helm-chart/templates/cvat_backend/worker_import/deployment.yml b/helm-chart/templates/cvat_backend/worker_import/deployment.yml index 9124fe70c52a..23178d95a4c4 100644 --- a/helm-chart/templates/cvat_backend/worker_import/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_import/deployment.yml @@ -54,7 +54,10 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.import"] + args: + - "run" + - "worker" + - "import" env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml b/helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml index 5c2ae121d3a3..8d2df200604f 100644 --- a/helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml @@ -54,7 +54,10 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.quality_reports"] + args: + - "run" + - "worker" + - "quality_reports" env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/utils/deployment.yml b/helm-chart/templates/cvat_backend/worker_utils/deployment.yml similarity index 91% rename from helm-chart/templates/cvat_backend/utils/deployment.yml rename to helm-chart/templates/cvat_backend/worker_utils/deployment.yml index 440f994b8b5a..0b02954676d9 100644 --- a/helm-chart/templates/cvat_backend/utils/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_utils/deployment.yml @@ -1,14 +1,14 @@ -{{- $localValues := .Values.cvat.backend.utils -}} +{{- $localValues := .Values.cvat.backend.worker.utils -}} apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Release.Name }}-backend-utils + name: {{ .Release.Name }}-backend-worker-utils namespace: {{ .Release.Namespace }} labels: app: cvat-app tier: backend - component: utils + component: worker-utils {{- include "cvat.labels" . | nindent 4 }} {{- with merge $localValues.labels .Values.cvat.backend.labels }} {{- toYaml . | nindent 4 }} @@ -29,13 +29,13 @@ spec: {{- end }} app: cvat-app tier: backend - component: utils + component: worker-utils template: metadata: labels: app: cvat-app tier: backend - component: utils + component: worker-utils {{- include "cvat.labels" . | nindent 8 }} {{- with merge $localValues.labels .Values.cvat.backend.labels }} {{- toYaml . | nindent 8 }} @@ -54,7 +54,14 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "utils"] + args: + - "run" + - "worker" + - "notifications" + - "cleaning" + {{- with $localValues.extraArgs }} + {{- toYaml . | nindent 10 }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_webhooks/deployment.yml b/helm-chart/templates/cvat_backend/worker_webhooks/deployment.yml index 46209eb1d578..69f72f0f282c 100644 --- a/helm-chart/templates/cvat_backend/worker_webhooks/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_webhooks/deployment.yml @@ -54,7 +54,10 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.webhooks"] + args: + - "run" + - "worker" + - "webhooks" env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/test.values.yaml b/helm-chart/test.values.yaml index bbdd843e9a22..4540a7acd66c 100644 --- a/helm-chart/test.values.yaml +++ b/helm-chart/test.values.yaml @@ -12,26 +12,38 @@ cvat: - name: DJANGO_SETTINGS_MODULE value: cvat.settings.testing_rest worker: - import: - replicas: 1 - additionalVolumeMounts: - - mountPath: /home/django/share - name: cvat-backend-data - subPath: share - export: - replicas: 1 + additionalEnv: + - name: DJANGO_SETTINGS_MODULE + value: cvat.settings.testing_rest + annotation: + replicas: 0 chunks: + replicas: 0 + consensus: + replicas: 0 + export: + replicas: 0 + import: + replicas: 0 + qualityreports: + replicas: 0 + webhooks: + replicas: 0 + utils: replicas: 1 additionalVolumeMounts: - mountPath: /home/django/share name: cvat-backend-data subPath: share - utils: - additionalEnv: - - name: DJANGO_SETTINGS_MODULE - value: cvat.settings.testing_rest - annotation: - replicas: 0 + extraArgs: + - "annotation" + - "chunks" + - "consensus" + - "export" + - "import" + - "quality_reports" + - "webhooks" + # Images are already present in the node imagePullPolicy: Never frontend: @@ -53,4 +65,3 @@ traefik: level: DEBUG access: enabled: true - diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index ba000f143277..45f35f525837 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -132,16 +132,17 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] - utils: - replicas: 1 - labels: {} - annotations: {} - resources: {} - affinity: {} - tolerations: [] - additionalEnv: [] - additionalVolumes: [] - additionalVolumeMounts: [] + utils: + replicas: 1 + labels: {} + annotations: {} + resources: {} + affinity: {} + tolerations: [] + additionalEnv: [] + additionalVolumes: [] + additionalVolumeMounts: [] + extraArgs: [] replicas: 1 image: cvat/server tag: dev diff --git a/supervisord/reusable/clamav.conf b/supervisord/reusable/clamav.conf new file mode 100644 index 000000000000..7324409c14f9 --- /dev/null +++ b/supervisord/reusable/clamav.conf @@ -0,0 +1,4 @@ +[program:clamav-update] +command=/usr/bin/freshclam -d -l %(ENV_HOME)s/logs/freshclam.log --foreground=true +numprocs=1 +startsecs=0 diff --git a/supervisord/reusable/rqscheduler.conf b/supervisord/reusable/rqscheduler.conf new file mode 100644 index 000000000000..442556a63fd9 --- /dev/null +++ b/supervisord/reusable/rqscheduler.conf @@ -0,0 +1,11 @@ +[program:rqscheduler] +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/rqscheduler.py + --host "%(ENV_CVAT_REDIS_INMEM_HOST)s" + --port "%(ENV_CVAT_REDIS_INMEM_PORT)s" + --password "%(ENV_CVAT_REDIS_INMEM_PASSWORD)s" + -i 30 + --path %(ENV_HOME)s +environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" +numprocs=1 +autorestart=true diff --git a/supervisord/reusable/smokescreen.conf b/supervisord/reusable/smokescreen.conf new file mode 100644 index 000000000000..67efb04db932 --- /dev/null +++ b/supervisord/reusable/smokescreen.conf @@ -0,0 +1,2 @@ +[program:smokescreen] +command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.quality_reports.conf b/supervisord/reusable/supervisord.conf similarity index 58% rename from supervisord/worker.quality_reports.conf rename to supervisord/reusable/supervisord.conf index a3c2e9820014..3ea5cc768c80 100644 --- a/supervisord/worker.quality_reports.conf +++ b/supervisord/reusable/supervisord.conf @@ -15,12 +15,3 @@ logfile_maxbytes=50MB ; maximum size of logfile before rotation logfile_backups=10 ; number of backed up logfiles loglevel=debug ; info, debug, warn, trace pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-quality-reports] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 quality_reports - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:quality_reports" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true diff --git a/supervisord/server.conf b/supervisord/server.conf index 7db00b7d2542..1afe17920d83 100644 --- a/supervisord/server.conf +++ b/supervisord/server.conf @@ -1,25 +1,5 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:clamav_update] -startsecs=0 -command=bash -c "if [ \"${CLAM_AV}\" = 'yes' ]; then /usr/bin/freshclam -d \ - -l %(ENV_HOME)s/logs/freshclam.log --foreground=true; fi" +[include] +files = reusable/supervisord.conf %(ENV_CVAT_SUPERVISORD_INCLUDES)s [program:nginx] command=/usr/sbin/nginx -g "daemon off;" @@ -37,9 +17,6 @@ command=%(ENV_HOME)s/wait_for_deps.sh --forwarded-allow-ips='*' cvat.asgi:application autorestart=true -environment=CVAT_EVENTS_LOCAL_DB_FILENAME="events_%(process_num)03d.db",CVAT_POSTGRES_APPLICATION_NAME="cvat:server" +environment=CVAT_EVENTS_LOCAL_DB_FILENAME="events_%(process_num)03d.db" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)d - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/utils.conf b/supervisord/utils.conf deleted file mode 100644 index dc7030023c35..000000000000 --- a/supervisord/utils.conf +++ /dev/null @@ -1,35 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqscheduler] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/rqscheduler.py - --host "%(ENV_CVAT_REDIS_INMEM_HOST)s" --port "%(ENV_CVAT_REDIS_INMEM_PORT)s" - --password "%(ENV_CVAT_REDIS_INMEM_PASSWORD)s" - -i 30 --path %(ENV_HOME)s -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" -numprocs=1 -autorestart=true - -[program:rqworker] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 notifications cleaning - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:notifications+cleaning" -numprocs=%(ENV_NUMPROCS)s -autorestart=true diff --git a/supervisord/worker.annotation.conf b/supervisord/worker.annotation.conf deleted file mode 100644 index de0060dafb0d..000000000000 --- a/supervisord/worker.annotation.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-annotation] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 annotation - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:annotation" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.chunks.conf b/supervisord/worker.chunks.conf deleted file mode 100644 index 9eccd41e8cba..000000000000 --- a/supervisord/worker.chunks.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-chunks] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 chunks - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:chunks" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.conf b/supervisord/worker.conf new file mode 100644 index 000000000000..df84aef498d5 --- /dev/null +++ b/supervisord/worker.conf @@ -0,0 +1,11 @@ +[include] +files = reusable/supervisord.conf %(ENV_CVAT_SUPERVISORD_INCLUDES)s + +[program:rqworker] +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 %(ENV_CVAT_QUEUES)s + --worker-class cvat.rqworker.DefaultWorker +environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" +numprocs=%(ENV_NUMPROCS)s +process_name=%(program_name)s-%(process_num)d +autorestart=true diff --git a/supervisord/worker.consensus.conf b/supervisord/worker.consensus.conf deleted file mode 100644 index 1072226b7786..000000000000 --- a/supervisord/worker.consensus.conf +++ /dev/null @@ -1,27 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location -childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live - -[program:rqworker-consensus] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 consensus - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:consensus" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true diff --git a/supervisord/worker.export.conf b/supervisord/worker.export.conf deleted file mode 100644 index 623bee37f186..000000000000 --- a/supervisord/worker.export.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-export] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 export - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:export" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.import.conf b/supervisord/worker.import.conf deleted file mode 100644 index de85e0d8ecde..000000000000 --- a/supervisord/worker.import.conf +++ /dev/null @@ -1,35 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-import] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 import - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:import" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:clamav-update] -command=bash -c "if [ \"${CLAM_AV}\" = 'yes' ]; then /usr/bin/freshclam -d \ - -l %(ENV_HOME)s/logs/freshclam.log --foreground=true; fi" -numprocs=1 -startsecs=0 - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.webhooks.conf b/supervisord/worker.webhooks.conf deleted file mode 100644 index e3401b48b95b..000000000000 --- a/supervisord/worker.webhooks.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-webhooks] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 webhooks - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:webhooks" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/tests/python/shared/fixtures/init.py b/tests/python/shared/fixtures/init.py index dac447a535a9..71ed2969ce81 100644 --- a/tests/python/shared/fixtures/init.py +++ b/tests/python/shared/fixtures/init.py @@ -42,7 +42,7 @@ class Container(str, Enum): WORKER_EXPORT = "cvat_worker_export" WORKER_QUALITY_REPORTS = "cvat_worker_quality_reports" WORKER_WEBHOOKS = "cvat_worker_webhooks" - UTILS = "cvat_utils" + WORKER_UTILS = "cvat_worker_utils" def __str__(self): return self.value @@ -326,7 +326,7 @@ def create_compose_files(container_name_files): for service_name, service_config in dc_config["services"].items(): service_config.pop("container_name", None) - if service_name in (Container.SERVER, Container.UTILS): + if service_name in (Container.SERVER, Container.WORKER_UTILS): service_env = service_config["environment"] service_env["DJANGO_SETTINGS_MODULE"] = "cvat.settings.testing_rest"