feat: improve uWSGI options

- Restart a worker after 1000 requests (max-requests)
- Restart a worker every hour (max-worker-lifetime)
- Restart a worker after 300MB resident memory usage (reload-on-rss)
- Restart a worker by force after 60 seconds (worker-reload-mercy)
- Use strict mode to catch unknown uWSGI options (strict)
- Require an application module to start (need-app)
- Avoid multiple interpreters (single-interpreter)
- Only use processes for concurrency, so only run one thread per
  process
- Suppress errors about clients closing sockets
- Add Harakiri options
- Add Sentry-setup
- Add uwsgitop settings

Refs: PT-1937

Author: mhieta

.prod/uwsgi.ini

@@ -1,22 +1,62 @@
 [uwsgi]
+# https://uwsgi-docs.readthedocs.io/en/latest/Options.html
+strict = true  # Fail if any option is unknown
+
 http-socket = :8000
+http-timeout = 60
 chdir = /app
 module = palvelutarjotin.wsgi
 static-map = /static=/var/static
 static-map = /media=/app/var/media
 uid = appuser
 gid = appuser
 
+# Settings required for uwsgitop
+stats = /tmp/statsock
+memory-report = true
+
+master = true
+# Enable threads for sentry, see:
+# https://docs.sentry.io/clients/python/advanced/#a-note-on-uwsgi
+enable-threads = true
+py-call-uwsgi-fork-hooks = true
+single-interpreter = true
+need-app = true
+processes = $(UWSGI_PROCESSES)
+threads = 1
+buffer-size = 65535 # Allow bigger requests that includes a big list of AD-groups. Default is 4096.
+
+# by default uwsgi reloads on SIGTERM instead of terminating
+# this makes container slow to stop, so we change it here
+die-on-term = true
+
+harakiri = 20
+harakiri-graceful-timeout = 5
+# Default listen queue is 100
+harakiri-queue-threshold = $(UWSGI_PROCESSES)
+
 # Reload workers regularly to keep memory fresh
 # and ease potential memory leaks
-max-requests = 1000                  # Restart workers after this many requests
-reload-on-rss = 300                  # Restart workers after this much resident memory
-worker-reload-mercy = 60             # How long to wait before forcefully killing workers (default is 60)
+max-requests = 1000         # Restart workers after this many requests
+max-worker-lifetime = 3600  # Restart workers after this many seconds
+reload-on-rss = 300         # Restart workers after this much resident memory
+worker-reload-mercy = 60    # How long to wait before forcefully killing workers (default is 60)
+
+# Suppress errors about clients closing sockets, happens with nginx as the ingress when
+# http pipes are closed before workers has had the time to serve content to the pipe
+ignore-sigpipe = true
+ignore-write-errors = true
+disable-write-exception = true
+
+if-env = SENTRY_DSN
+print = Enabled sentry logging for uWSGI
+plugin = sentry
+alarm = logsentry sentry:dsn=$(SENTRY_DSN),logger=uwsgi.sentry
+# Log full queue, segfault and harakiri errors to sentry
+alarm-backlog = logsentry
+alarm-segfault = logsentry
+alarm-log = logsentry HARAKIRI \[core.*\]
+endif =
 
-master = 1
-processes = 2
-threads = 2
 route = ^/readiness$ donotlog:
 route = ^/healthz$ donotlog:
-
-buffer-size = 65535 # Allow bigger requests that includes a big list of AD-groups. Default is 4096.

Dockerfile

@@ -18,7 +18,8 @@ RUN yum update -y && yum install -y \
     nc \
     && pip install -U pip \
     && pip install --no-cache-dir -r /app/requirements.txt \
-    && pip install --no-cache-dir  -r /app/requirements-prod.txt
+    && pip install --no-cache-dir  -r /app/requirements-prod.txt \
+    && yum clean all
 
 # fatal: detected dubious ownership in repository at '/app'
 RUN git config --system --add safe.directory /app

docker-entrypoint.sh

@@ -23,5 +23,7 @@ if [[ ! -z "$@" ]]; then
 elif [[ "$DEV_SERVER" = "1" ]]; then
     python ./manage.py runserver 0.0.0.0:8081
 else
+    export UWSGI_PROCESSES=${UWSGI_PROCESSES:-4}
+    uwsgi --build-plugin https://github.com/City-of-Helsinki/uwsgi-sentry
     uwsgi --ini .prod/uwsgi.ini
 fi