diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 93a9ae35fc8..5d3ea575acb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -474,9 +474,7 @@ jobs: run: | mkdir ${LOGS_DIR} kubectl logs $(kubectl get pods -l component=server -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_server.log - kubectl logs $(kubectl get pods -l component=worker-import -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_import.log - kubectl logs $(kubectl get pods -l component=worker-export -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_export.log - kubectl logs $(kubectl get pods -l component=worker-webhooks -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_worker_webhooks.log + kubectl logs $(kubectl get pods -l component=worker-utils -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/cvat_workers.log kubectl logs $(kubectl get pods -l app.kubernetes.io/name=traefik -o 'jsonpath={.items[0].metadata.name}') >${LOGS_DIR}/traefik.log - name: Uploading "cvat" container logs as an artifact diff --git a/backend_entrypoint.sh b/backend_entrypoint.sh index 5b7f24ebe7e..4b7cb0e3aae 100755 --- a/backend_entrypoint.sh +++ b/backend_entrypoint.sh @@ -28,12 +28,42 @@ cmd_init() { ~/manage.py syncperiodicjobs } +_get_supervisord_includes() { + extra_configs="" + for arg in "$@"; do + if [[ "$arg" = include=* ]]; then + extra_configs+=" ${arg#include=}_reusable.conf" + fi + done + echo $extra_configs +} + +_get_worker_list() { + workers="" + for arg in "$@"; do + if [[ "$arg" != include=* ]]; then + workers+=" $arg" + fi + done + echo $workers +} + +_get_app_label() { + label=$1 + for arg in "${@:2}"; do + label+="+${arg}" + done + echo $label +} + cmd_run() { - if [ "$#" -ne 1 ]; then - fail "run: expected 1 argument" + if [ "$#" -eq 0 ]; then + fail "run: at least 1 argument is expected" fi - if [ "$1" = "server" ]; then + component="$1" + + if [ "$component" = "server" ]; then ~/manage.py collectstatic --no-input fi @@ -50,7 +80,24 @@ cmd_run() { sleep 10 done - exec supervisord -c "supervisord/$1.conf" + supervisord_includes=$(_get_supervisord_includes "${@:2}") + postgres_app_name="cvat:$component" + + if [ "$component" = "worker" ]; then + if [ "$#" = 1 ]; then + fail "run worker: expected at least 1 worker name" + fi + + worker_list=$(_get_worker_list "${@:2}") + echo "Workers to run: $worker_list" + export CVAT_WORKERS=$worker_list + postgres_app_name+=":$(_get_app_label $worker_list)" + fi + + export CVAT_POSTGRES_APPLICATION_NAME=$postgres_app_name + export CVAT_SUPERVISORD_INCLUDES=$supervisord_includes + + exec supervisord -c "supervisord/$component.conf" } if [ $# -eq 0 ]; then @@ -59,7 +106,8 @@ if [ $# -eq 0 ]; then echo >&2 "available subcommands:" echo >&2 " bash " echo >&2 " init" - echo >&2 " run " + echo >&2 " run server " + echo >&2 " run worker " exit 1 fi diff --git a/docker-compose.external_db.yml b/docker-compose.external_db.yml index 24d94e2f256..7b149d8e109 100644 --- a/docker-compose.external_db.yml +++ b/docker-compose.external_db.yml @@ -20,7 +20,7 @@ services: replicas: 0 cvat_server: *backend-settings - cvat_utils: *backend-settings + cvat_worker_utils: *backend-settings cvat_worker_analytics_reports: *backend-settings cvat_worker_annotation: *backend-settings cvat_worker_export: *backend-settings diff --git a/docker-compose.yml b/docker-compose.yml index a5c96bd0b68..a4bbdf86053 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -97,7 +97,7 @@ services: CVAT_ANALYTICS: 1 CVAT_BASE_URL: ONE_RUNNING_JOB_IN_QUEUE_PER_USER: - command: init run server + command: init run server include=clamav include=smokescreen labels: traefik.enable: "true" traefik.http.services.cvat.loadbalancer.server.port: "8080" @@ -114,8 +114,8 @@ services: aliases: - cvat-server - cvat_utils: - container_name: cvat_utils + cvat_worker_utils: + container_name: cvat_worker_utils image: cvat/server:${CVAT_VERSION:-dev} restart: always depends_on: *backend-deps @@ -123,7 +123,7 @@ services: <<: *backend-env CVAT_REDIS_INMEM_PASSWORD: '' NUMPROCS: 1 - command: run utils + command: run worker notifications cleaning include=rqscheduler volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -139,7 +139,7 @@ services: environment: <<: *backend-env NUMPROCS: 2 - command: run worker.import + command: run worker import include=smokescreen include=clamav volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -155,7 +155,7 @@ services: environment: <<: [*backend-env, *clickhouse-env] NUMPROCS: 2 - command: run worker.export + command: run worker export include=smokescreen volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -171,7 +171,7 @@ services: environment: <<: *backend-env NUMPROCS: 1 - command: run worker.annotation + command: run worker annotation include=smokescreen volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -187,7 +187,7 @@ services: environment: <<: *backend-env NUMPROCS: 1 - command: run worker.webhooks + command: run worker webhooks include=smokescreen volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -203,7 +203,7 @@ services: environment: <<: *backend-env NUMPROCS: 1 - command: run worker.quality_reports + command: run worker quality_reports volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -219,7 +219,7 @@ services: environment: <<: [*backend-env, *clickhouse-env] NUMPROCS: 2 - command: run worker.analytics_reports + command: run worker analytics_reports volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -235,7 +235,7 @@ services: environment: <<: *backend-env NUMPROCS: 2 - command: run worker.chunks + command: run worker chunks include=smokescreen volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys @@ -251,7 +251,7 @@ services: environment: <<: *backend-env NUMPROCS: 1 - command: run worker.consensus + command: run worker consensus volumes: - cvat_data:/home/django/data - cvat_keys:/home/django/keys diff --git a/helm-chart/Chart.yaml b/helm-chart/Chart.yaml index 2b8d11f50bf..d26860ccb8b 100644 --- a/helm-chart/Chart.yaml +++ b/helm-chart/Chart.yaml @@ -16,7 +16,7 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 0.14.4 +version: 0.15.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to diff --git a/helm-chart/templates/cvat_backend/server/deployment.yml b/helm-chart/templates/cvat_backend/server/deployment.yml index 28daf0da7cb..6c16bf3ce03 100644 --- a/helm-chart/templates/cvat_backend/server/deployment.yml +++ b/helm-chart/templates/cvat_backend/server/deployment.yml @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "server"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: - name: ALLOWED_HOSTS value: {{ $localValues.envs.ALLOWED_HOSTS | squote}} diff --git a/helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml b/helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml index c2f05429169..bca550418b8 100644 --- a/helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_analyticsreports/deployment.yml @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.analytics_reports"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{ include "cvat.sharedClickhouseEnv" . | indent 10 }} diff --git a/helm-chart/templates/cvat_backend/worker_annotation/deployment.yml b/helm-chart/templates/cvat_backend/worker_annotation/deployment.yml index a2439325ca4..da98f22f7c8 100644 --- a/helm-chart/templates/cvat_backend/worker_annotation/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_annotation/deployment.yml @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.annotation"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_chunks/deployment.yml b/helm-chart/templates/cvat_backend/worker_chunks/deployment.yml index 74e80b1b185..ae2dd8b6ad2 100644 --- a/helm-chart/templates/cvat_backend/worker_chunks/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_chunks/deployment.yml @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.chunks"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml b/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml index c96b5aa2e96..8e4ae608c82 100644 --- a/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_consensus/deployment.yml @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.consensus"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_export/deployment.yml b/helm-chart/templates/cvat_backend/worker_export/deployment.yml index 6cb755dbd01..86d4637936c 100644 --- a/helm-chart/templates/cvat_backend/worker_export/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_export/deployment.yml @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.export"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{ include "cvat.sharedClickhouseEnv" . | indent 10 }} diff --git a/helm-chart/templates/cvat_backend/worker_import/deployment.yml b/helm-chart/templates/cvat_backend/worker_import/deployment.yml index 9124fe70c52..51fc8615a74 100644 --- a/helm-chart/templates/cvat_backend/worker_import/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_import/deployment.yml @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.import"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml b/helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml index 5c2ae121d3a..f5234451510 100644 --- a/helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_qualityreports/deployment.yml @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.quality_reports"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/utils/deployment.yml b/helm-chart/templates/cvat_backend/worker_utils/deployment.yml similarity index 92% rename from helm-chart/templates/cvat_backend/utils/deployment.yml rename to helm-chart/templates/cvat_backend/worker_utils/deployment.yml index 440f994b8b5..d66f884132d 100644 --- a/helm-chart/templates/cvat_backend/utils/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_utils/deployment.yml @@ -1,14 +1,14 @@ -{{- $localValues := .Values.cvat.backend.utils -}} +{{- $localValues := .Values.cvat.backend.worker.utils -}} apiVersion: apps/v1 kind: Deployment metadata: - name: {{ .Release.Name }}-backend-utils + name: {{ .Release.Name }}-backend-worker-utils namespace: {{ .Release.Namespace }} labels: app: cvat-app tier: backend - component: utils + component: worker-utils {{- include "cvat.labels" . | nindent 4 }} {{- with merge $localValues.labels .Values.cvat.backend.labels }} {{- toYaml . | nindent 4 }} @@ -29,13 +29,13 @@ spec: {{- end }} app: cvat-app tier: backend - component: utils + component: worker-utils template: metadata: labels: app: cvat-app tier: backend - component: utils + component: worker-utils {{- include "cvat.labels" . | nindent 8 }} {{- with merge $localValues.labels .Values.cvat.backend.labels }} {{- toYaml . | nindent 8 }} @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "utils"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/templates/cvat_backend/worker_webhooks/deployment.yml b/helm-chart/templates/cvat_backend/worker_webhooks/deployment.yml index 46209eb1d57..d8882bd2917 100644 --- a/helm-chart/templates/cvat_backend/worker_webhooks/deployment.yml +++ b/helm-chart/templates/cvat_backend/worker_webhooks/deployment.yml @@ -54,7 +54,12 @@ spec: resources: {{- toYaml . | nindent 12 }} {{- end }} - args: ["run", "worker.webhooks"] + args: + {{- with $localValues.args }} + {{- range . }} + - {{ . | quote }} + {{- end }} + {{- end }} env: {{ include "cvat.sharedBackendEnv" . | indent 10 }} {{- with concat .Values.cvat.backend.additionalEnv $localValues.additionalEnv }} diff --git a/helm-chart/test.values.yaml b/helm-chart/test.values.yaml index bbdd843e9a2..70a40b9d96b 100644 --- a/helm-chart/test.values.yaml +++ b/helm-chart/test.values.yaml @@ -12,26 +12,64 @@ cvat: - name: DJANGO_SETTINGS_MODULE value: cvat.settings.testing_rest worker: - import: - replicas: 1 - additionalVolumeMounts: - - mountPath: /home/django/share - name: cvat-backend-data - subPath: share - export: - replicas: 1 + analytics_reports: + replicas: 0 + annotation: + replicas: 0 chunks: + replicas: 0 + consensus: + replicas: 0 + export: + replicas: 0 + import: + replicas: 0 + webhooks: + replicas: 0 + utils: replicas: 1 + additionalEnv: + - name: DJANGO_SETTINGS_MODULE + value: cvat.settings.testing_rest + - name: CLICKHOUSE_HOST + valueFrom: + secretKeyRef: + name: cvat-analytics-secret + key: CLICKHOUSE_HOST + - name: CLICKHOUSE_PORT + valueFrom: + secretKeyRef: + name: cvat-analytics-secret + key: CLICKHOUSE_PORT + - name: CLICKHOUSE_USER + valueFrom: + secretKeyRef: + name: cvat-analytics-secret + key: CLICKHOUSE_USER + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: cvat-analytics-secret + key: CLICKHOUSE_PASSWORD additionalVolumeMounts: - mountPath: /home/django/share name: cvat-backend-data subPath: share - utils: - additionalEnv: - - name: DJANGO_SETTINGS_MODULE - value: cvat.settings.testing_rest - annotation: - replicas: 0 + args: + - "run" + - "worker" + - "analytics_reports" + - "annotation" + - "chunks" + - "cleaning" + - "consensus" + - "export" + - "import" + - "notifications" + - "webhooks" + - "include=rqscheduler" + - "include=smokescreen" + # Images are already present in the node imagePullPolicy: Never frontend: diff --git a/helm-chart/values.yaml b/helm-chart/values.yaml index f05f03a1137..8b4fa852e51 100644 --- a/helm-chart/values.yaml +++ b/helm-chart/values.yaml @@ -51,6 +51,11 @@ cvat: periodSeconds: 15 failureThreshold: 5 initialDelaySeconds: 60 + args: + - "run" + - "server" + - "include=clamav" + - "include=smokescreen" worker: livenessProbe: enabled: true @@ -67,6 +72,11 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] + args: + - "run" + - "worker" + - "export" + - "include=smokescreen" import: replicas: 2 labels: {} @@ -77,6 +87,12 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] + args: + - "run" + - "worker" + - "import" + - "include=smokescreen" + - "include=clamav" annotation: replicas: 1 labels: {} @@ -87,6 +103,10 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] + args: + - "run" + - "worker" + - "annotation" webhooks: replicas: 1 labels: {} @@ -97,6 +117,11 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] + args: + - "run" + - "worker" + - "webhooks" + - "include=smokescreen" qualityreports: replicas: 1 labels: {} @@ -107,6 +132,10 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] + args: + - "run" + - "worker" + - "quality_reports" analyticsreports: replicas: 1 labels: {} @@ -117,6 +146,10 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] + args: + - "run" + - "worker" + - "analytics_reports" chunks: replicas: 2 labels: {} @@ -127,6 +160,11 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] + args: + - "run" + - "worker" + - "chunks" + - "include=smokescreen" consensus: replicas: 1 labels: {} @@ -137,16 +175,26 @@ cvat: additionalEnv: [] additionalVolumes: [] additionalVolumeMounts: [] - utils: - replicas: 1 - labels: {} - annotations: {} - resources: {} - affinity: {} - tolerations: [] - additionalEnv: [] - additionalVolumes: [] - additionalVolumeMounts: [] + args: + - "run" + - "worker" + - "consensus" + utils: + replicas: 1 + labels: {} + annotations: {} + resources: {} + affinity: {} + tolerations: [] + additionalEnv: [] + additionalVolumes: [] + additionalVolumeMounts: [] + args: + - "run" + - "worker" + - "notifications" + - "cleaning" + - "include=rqscheduler" replicas: 1 image: cvat/server tag: dev diff --git a/supervisord/clamav_reusable.conf b/supervisord/clamav_reusable.conf new file mode 100644 index 00000000000..18ceea7ee99 --- /dev/null +++ b/supervisord/clamav_reusable.conf @@ -0,0 +1,5 @@ +[program:clamav-update] +command=bash -c "if [ \"${CLAM_AV}\" = 'yes' ]; then /usr/bin/freshclam -d \ + -l %(ENV_HOME)s/logs/freshclam.log --foreground=true; fi" +numprocs=1 +startsecs=0 diff --git a/supervisord/rqscheduler_reusable.conf b/supervisord/rqscheduler_reusable.conf new file mode 100644 index 00000000000..442556a63fd --- /dev/null +++ b/supervisord/rqscheduler_reusable.conf @@ -0,0 +1,11 @@ +[program:rqscheduler] +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/rqscheduler.py + --host "%(ENV_CVAT_REDIS_INMEM_HOST)s" + --port "%(ENV_CVAT_REDIS_INMEM_PORT)s" + --password "%(ENV_CVAT_REDIS_INMEM_PASSWORD)s" + -i 30 + --path %(ENV_HOME)s +environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" +numprocs=1 +autorestart=true diff --git a/supervisord/server.conf b/supervisord/server.conf index 7db00b7d254..e9d16a28b6f 100644 --- a/supervisord/server.conf +++ b/supervisord/server.conf @@ -1,25 +1,5 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:clamav_update] -startsecs=0 -command=bash -c "if [ \"${CLAM_AV}\" = 'yes' ]; then /usr/bin/freshclam -d \ - -l %(ENV_HOME)s/logs/freshclam.log --foreground=true; fi" +[include] +files = supervisord_reusable.conf %(ENV_CVAT_SUPERVISORD_INCLUDES)s [program:nginx] command=/usr/sbin/nginx -g "daemon off;" @@ -37,9 +17,6 @@ command=%(ENV_HOME)s/wait_for_deps.sh --forwarded-allow-ips='*' cvat.asgi:application autorestart=true -environment=CVAT_EVENTS_LOCAL_DB_FILENAME="events_%(process_num)03d.db",CVAT_POSTGRES_APPLICATION_NAME="cvat:server" +environment=CVAT_EVENTS_LOCAL_DB_FILENAME="events_%(process_num)03d.db" numprocs=%(ENV_NUMPROCS)s process_name=%(program_name)s-%(process_num)d - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/smokescreen_reusable.conf b/supervisord/smokescreen_reusable.conf new file mode 100644 index 00000000000..67efb04db93 --- /dev/null +++ b/supervisord/smokescreen_reusable.conf @@ -0,0 +1,2 @@ +[program:smokescreen] +command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.quality_reports.conf b/supervisord/supervisord_reusable.conf similarity index 58% rename from supervisord/worker.quality_reports.conf rename to supervisord/supervisord_reusable.conf index a3c2e982001..3ea5cc768c8 100644 --- a/supervisord/worker.quality_reports.conf +++ b/supervisord/supervisord_reusable.conf @@ -15,12 +15,3 @@ logfile_maxbytes=50MB ; maximum size of logfile before rotation logfile_backups=10 ; number of backed up logfiles loglevel=debug ; info, debug, warn, trace pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-quality-reports] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 quality_reports - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:quality_reports" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true diff --git a/supervisord/utils.conf b/supervisord/utils.conf deleted file mode 100644 index dc7030023c3..00000000000 --- a/supervisord/utils.conf +++ /dev/null @@ -1,35 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqscheduler] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/rqscheduler.py - --host "%(ENV_CVAT_REDIS_INMEM_HOST)s" --port "%(ENV_CVAT_REDIS_INMEM_PORT)s" - --password "%(ENV_CVAT_REDIS_INMEM_PASSWORD)s" - -i 30 --path %(ENV_HOME)s -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" -numprocs=1 -autorestart=true - -[program:rqworker] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 notifications cleaning - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:notifications+cleaning" -numprocs=%(ENV_NUMPROCS)s -autorestart=true diff --git a/supervisord/worker.analytics_reports.conf b/supervisord/worker.analytics_reports.conf deleted file mode 100644 index 580828a58d6..00000000000 --- a/supervisord/worker.analytics_reports.conf +++ /dev/null @@ -1,26 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-analytics-reports] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 analytics_reports - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:analytics_reports" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true diff --git a/supervisord/worker.annotation.conf b/supervisord/worker.annotation.conf deleted file mode 100644 index de0060dafb0..00000000000 --- a/supervisord/worker.annotation.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-annotation] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 annotation - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:annotation" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.chunks.conf b/supervisord/worker.chunks.conf deleted file mode 100644 index 9eccd41e8cb..00000000000 --- a/supervisord/worker.chunks.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-chunks] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 chunks - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:chunks" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.conf b/supervisord/worker.conf new file mode 100644 index 00000000000..922f0ede24a --- /dev/null +++ b/supervisord/worker.conf @@ -0,0 +1,11 @@ +[include] +files = supervisord_reusable.conf %(ENV_CVAT_SUPERVISORD_INCLUDES)s + +[program:rqworker] +command=%(ENV_HOME)s/wait_for_deps.sh + python3 %(ENV_HOME)s/manage.py rqworker -v 3 %(ENV_CVAT_WORKERS)s + --worker-class cvat.rqworker.DefaultWorker +environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler" +numprocs=%(ENV_NUMPROCS)s +process_name=%(program_name)s-%(process_num)d +autorestart=true diff --git a/supervisord/worker.consensus.conf b/supervisord/worker.consensus.conf deleted file mode 100644 index 1072226b778..00000000000 --- a/supervisord/worker.consensus.conf +++ /dev/null @@ -1,27 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location -childlogdir=%(ENV_HOME)s/logs/ ; where child log files will live - -[program:rqworker-consensus] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 consensus - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:consensus" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true diff --git a/supervisord/worker.export.conf b/supervisord/worker.export.conf deleted file mode 100644 index 623bee37f18..00000000000 --- a/supervisord/worker.export.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-export] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 export - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:export" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.import.conf b/supervisord/worker.import.conf deleted file mode 100644 index de85e0d8ecd..00000000000 --- a/supervisord/worker.import.conf +++ /dev/null @@ -1,35 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-import] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 import - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:import" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:clamav-update] -command=bash -c "if [ \"${CLAM_AV}\" = 'yes' ]; then /usr/bin/freshclam -d \ - -l %(ENV_HOME)s/logs/freshclam.log --foreground=true; fi" -numprocs=1 -startsecs=0 - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/supervisord/worker.webhooks.conf b/supervisord/worker.webhooks.conf deleted file mode 100644 index e3401b48b95..00000000000 --- a/supervisord/worker.webhooks.conf +++ /dev/null @@ -1,29 +0,0 @@ -[unix_http_server] -file = /tmp/supervisord/supervisor.sock - -[supervisorctl] -serverurl = unix:///tmp/supervisord/supervisor.sock - - -[rpcinterface:supervisor] -supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface - -[supervisord] -nodaemon=true -logfile=%(ENV_HOME)s/logs/supervisord.log ; supervisord log file -logfile_maxbytes=50MB ; maximum size of logfile before rotation -logfile_backups=10 ; number of backed up logfiles -loglevel=debug ; info, debug, warn, trace -pidfile=/tmp/supervisord/supervisord.pid ; pidfile location - -[program:rqworker-webhooks] -command=%(ENV_HOME)s/wait_for_deps.sh - python3 %(ENV_HOME)s/manage.py rqworker -v 3 webhooks - --worker-class cvat.rqworker.DefaultWorker -environment=VECTOR_EVENT_HANDLER="SynchronousLogstashHandler",CVAT_POSTGRES_APPLICATION_NAME="cvat:worker:webhooks" -numprocs=%(ENV_NUMPROCS)s -process_name=%(program_name)s-%(process_num)d -autorestart=true - -[program:smokescreen] -command=smokescreen --listen-ip=127.0.0.1 %(ENV_SMOKESCREEN_OPTS)s diff --git a/tests/python/shared/fixtures/init.py b/tests/python/shared/fixtures/init.py index 93e4d72758d..b6438c021fe 100644 --- a/tests/python/shared/fixtures/init.py +++ b/tests/python/shared/fixtures/init.py @@ -42,7 +42,7 @@ class Container(str, Enum): WORKER_EXPORT = "cvat_worker_export" WORKER_QUALITY_REPORTS = "cvat_worker_quality_reports" WORKER_WEBHOOKS = "cvat_worker_webhooks" - UTILS = "cvat_utils" + WORKER_UTILS = "cvat_worker_utils" def __str__(self): return self.value @@ -326,7 +326,7 @@ def create_compose_files(container_name_files): for service_name, service_config in dc_config["services"].items(): service_config.pop("container_name", None) - if service_name in (Container.SERVER, Container.UTILS): + if service_name in (Container.SERVER, Container.WORKER_UTILS): service_env = service_config["environment"] service_env["DJANGO_SETTINGS_MODULE"] = "cvat.settings.testing_rest"