Add configurable connection pool for Redis-py

- Redis was using a default connection pool, but not tuned or configured
- Log every ignored Redis exception
- Use redis[hiredis] for dependencies, so redis-py can choose the best hiredis version

Author: Uxio0

config/settings/base.py

@@ -529,6 +529,9 @@
 }
 
 REDIS_URL = env("REDIS_URL", default="redis://localhost:6379/0")
+REDIS_TIMEOUT_SECONDS = env("REDIS_TIMEOUT_SECONDS", default=5)
+REDIS_CONNECTION_TIMEOUT_SECONDS = env("REDIS_CONNECTION_TIMEOUT_SECONDS", default=5)
+REDIS_POOL_MAX_CONNECTIONS = env("REDIS_POOL_MAX_CONNECTIONS", default=300)
 
 # Ethereum RPC
 # ------------------------------------------------------------------------------

config/settings/local.py

@@ -1,5 +1,12 @@
 from .base import *  # noqa
 from .base import env
+from .base import (
+    REDIS_URL,
+    REDIS_CONNECTION_TIMEOUT_SECONDS,
+    REDIS_TIMEOUT_SECONDS,
+    REDIS_POOL_MAX_CONNECTIONS,
+)
+
 
 # GENERAL
 # ------------------------------------------------------------------------------
@@ -13,8 +20,6 @@
 # https://docs.djangoproject.com/en/dev/ref/settings/#allowed-hosts
 ALLOWED_HOSTS = env.list("DJANGO_ALLOWED_HOSTS", default=["*"])
 
-REDIS_URL = env.str("REDIS_URL")
-
 # SECURITY
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/middleware/#x-content-type-options-nosniff
@@ -29,6 +34,9 @@
         "LOCATION": REDIS_URL,
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            "SOCKET_CONNECT_TIMEOUT": REDIS_CONNECTION_TIMEOUT_SECONDS,
+            "SOCKET_TIMEOUT": REDIS_TIMEOUT_SECONDS,
+            "CONNECTION_POOL_KWARGS": {"max_connections": REDIS_POOL_MAX_CONNECTIONS},
             # Mimicking memcache behavior.
             # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
             "IGNORE_EXCEPTIONS": True,
@@ -40,6 +48,9 @@
     },
 }
 
+# Log redis exceptions ignored
+DJANGO_REDIS_LOG_IGNORED_EXCEPTIONS = True
+
 # django-debug-toolbar
 # ------------------------------------------------------------------------------
 # https://django-debug-toolbar.readthedocs.io/en/latest/installation.html#prerequisites

config/settings/production.py

@@ -1,5 +1,11 @@
 from .base import *  # noqa
 from .base import env
+from .base import (
+    REDIS_URL,
+    REDIS_CONNECTION_TIMEOUT_SECONDS,
+    REDIS_TIMEOUT_SECONDS,
+    REDIS_POOL_MAX_CONNECTIONS,
+)
 
 # GENERAL
 # ------------------------------------------------------------------------------
@@ -13,16 +19,18 @@
 # DATABASES['default'] = env.db('DATABASE_URL')  # noqa F405
 DATABASES["default"]["ATOMIC_REQUESTS"] = False  # noqa F405
 
-REDIS_URL = env.str("REDIS_URL")
-
 # CACHES
 # ------------------------------------------------------------------------------
+# https://docs.djangoproject.com/en/dev/ref/settings/#caches
 CACHES = {
     "default": {
         "BACKEND": "django_redis.cache.RedisCache",
         "LOCATION": REDIS_URL,
         "OPTIONS": {
             "CLIENT_CLASS": "django_redis.client.DefaultClient",
+            "SOCKET_CONNECT_TIMEOUT": REDIS_CONNECTION_TIMEOUT_SECONDS,
+            "SOCKET_TIMEOUT": REDIS_TIMEOUT_SECONDS,
+            "CONNECTION_POOL_KWARGS": {"max_connections": REDIS_POOL_MAX_CONNECTIONS},
             # Mimicking memcache behavior.
             # http://niwinz.github.io/django-redis/latest/#_memcached_exceptions_behavior
             "IGNORE_EXCEPTIONS": True,
@@ -34,6 +42,9 @@
     },
 }
 
+# Log redis exceptions ignored
+DJANGO_REDIS_LOG_IGNORED_EXCEPTIONS = True
+
 # SECURITY
 # ------------------------------------------------------------------------------
 # https://docs.djangoproject.com/en/dev/ref/middleware/#x-content-type-options-nosniff

requirements.txt

@@ -21,12 +21,11 @@ docutils==0.22.2
 drf-spectacular==0.28.0
 flower==2.0.1
 gunicorn[gevent]==23.0.0
-hiredis==3.2.1
 packaging>=21.0
 pika==1.3.2
 pillow==11.3.0
 psycopg[binary,pool]==3.2.12
-redis==6.4.0
+redis[hiredis]==6.4.0
 requests==2.32.5
 safe-eth-py[django]==7.14.0
 web3==7.14.0

safe_transaction_service/utils/redis.py

@@ -4,7 +4,7 @@
 
 from django.conf import settings
 
-from redis import Redis
+from redis import ConnectionPool, Redis
 
 logger = logging.getLogger(__name__)
 
@@ -16,4 +16,12 @@ def get_redis() -> Redis:
     # Encode memoryview for redis when using pickle
     copyreg.pickle(memoryview, lambda val: (memoryview, (bytes(val),)))
 
-    return Redis.from_url(settings.REDIS_URL)
+    connection_pool = ConnectionPool(
+        max_connections=settings.REDIS_POOL_MAX_CONNECTIONS
+    ).from_url(
+        settings.REDIS_URL,
+        socket_connect_timeout=settings.REDIS_CONNECTION_TIMEOUT_SECONDS,
+        socket_timeout=settings.REDIS_TIMEOUT_SECONDS,
+        health_check_interval=30,
+    )
+    return Redis(connection_pool=connection_pool)