Merge pull request #94 from 101t/3.0.3

3.0.3

Author: 101t

.travis.yml

@@ -8,7 +8,7 @@ before_install:
   - cp -rf sample.env .env
 before_script:
   # install requirements:
-  - pip install -U pip wheel uv && uv pip install -r pyproject.toml
+  - pip install -U pip wheel uv && uv pip install -r pyproject.toml --extra prod
 services:
   - redis-server
 addons:

README.md

@@ -159,9 +159,11 @@ TELNET_USERNAME=jcliadmin
 TELNET_PW=jclipwd
 TELNET_TIMEOUT=10
 
-# Redis & Celery
-REDIS_URL=redis://localhost:6379/0
-CELERY_BROKER_URL=amqp://guest:guest@localhost:5672//
+# Redis for Cache & Celery
+REDIS_HOST=redis
+REDIS_PORT=6379
+REDIS_DB=0
+REDIS_PASSWORD=<Optional>
 
 # Submit Log Feature
 SUBMIT_LOG=True
@@ -284,8 +286,9 @@ For ARM-based systems:
 | `SECRET_KEY` | Django secret key | - | ✅ |
 | `ALLOWED_HOSTS` | Allowed hosts | `*` | ✅ |
 | `PRODB_URL` | PostgreSQL URL | - | ✅ |
-| `REDIS_URL` | Redis URL | `redis://redis:6379/0` | ✅ |
-| `CELERY_BROKER_URL` | RabbitMQ URL | `amqp://guest:guest@rabbit-mq:5672//` | ✅ |
+| `REDIS_HOST` | Redis host | `redis` | ✅ |
+| `REDIS_PORT` | Redis port | `6379` | ✅ |
+| `REDIS_DB` | Redis database | `0` | ✅ |
 | `TELNET_HOST` | Jasmin telnet host | `127.0.0.1` | ✅ |
 | `TELNET_PORT` | Jasmin telnet port | `8990` | ✅ |
 | `TELNET_USERNAME` | Jasmin admin username | `jcliadmin` | ✅ |

config/celery.py

@@ -1,15 +1,19 @@
 import os
+import django
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.pro")
+
+# Setup Django before importing any Django modules
+django.setup()
+
 from django.conf import settings
 from celery import Celery
 from celery.utils.log import get_task_logger
-
 from django.utils import timezone
 
 logger = get_task_logger(__name__)
-
-os.environ.setdefault("DJANGO_SETTINGS_MODULE", "config.settings.pro")
-CELERY_BROKER_URL = os.environ.get("CELERY_BROKER_URL", default="redis://localhost:6379/0")
-CELERY_RESULT_BACKEND = os.environ.get("CELERY_RESULT_BACKEND", default="redis://localhost:6379/0")
+CELERY_BROKER_URL = settings.REDIS_URL
+CELERY_RESULT_BACKEND = settings.REDIS_URL
 
 
 app = Celery('config')
@@ -21,6 +25,20 @@
 app.conf.broker_connection_retry_on_startup = True
 app.autodiscover_tasks(lambda: settings.INSTALLED_APPS, related_name='tasks')
 
+# Close database connections after each task to prevent leaks
+from django.db import connection
+from celery.signals import task_postrun, task_prerun
+
+@task_prerun.connect
+def task_prerun_handler(sender=None, **kwargs):
+    """Close database connections before task starts."""
+    connection.close()
+
+@task_postrun.connect
+def task_postrun_handler(sender=None, **kwargs):
+    """Close database connections after task completes."""
+    connection.close()
+
 BROKER_CONNECTION_TIMEOUT = 120
 
 CELERY_DEFAULT_UP_TIME = timezone.now()

config/settings/com.py

@@ -144,7 +144,20 @@
 REDIS_HOST = os.environ.get("REDIS_HOST", default="redis")
 REDIS_PORT = int(os.environ.get("REDIS_PORT", default=6379))
 REDIS_DB = int(os.environ.get("REDIS_DB", default=0))
-REDIS_URL = (REDIS_HOST, REDIS_PORT)
+REDIS_PASSWORD = os.environ.get("REDIS_PASSWORD", default="")
+if REDIS_PASSWORD:
+    REDIS_URL = f'redis://:{REDIS_PASSWORD}@{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+else:
+    REDIS_URL = f'redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_DB}'
+
+CACHES = {
+    'default': {
+        'BACKEND': 'django.core.cache.backends.redis.RedisCache',
+        'LOCATION': REDIS_URL,
+        'KEY_PREFIX': 'jasmin_cache',
+        'TIMEOUT': 300,  # 5 minutes default
+    }
+}
 
 DEFAULT_USER_AVATAR = STATIC_URL + "assets/img/user.png"
 DEFAULT_USER_FOLDER = "users"

config/version.py

@@ -1 +1 @@
-VERSION = "3.0.1"
+VERSION = "3.0.3"

docker-compose.yml

@@ -2,7 +2,7 @@ version: '3.8'
 
 services:
   jasmin-web:
-    image: tarekaec/jasmin_web_panel:1.3
+    image: tarekaec/jasmin_web_panel:1.4.2
     ports:
       - "${JASMIN_WEB_PORT:-8999}:8000"
     deploy:
@@ -22,11 +22,11 @@ services:
     depends_on:
       - redis
       - db
-      - rabbit-mq
+      - rabbitmq
     restart: unless-stopped
 
   jasmin-celery:
-    image: tarekaec/jasmin_web_panel:1.3
+    image: tarekaec/jasmin_web_panel:1.4.2
     entrypoint: bash ./docker-entrypoint-celery.sh
     deploy:
       replicas: 1
@@ -35,8 +35,6 @@ services:
     environment:
       DEBUG: 0
       DJANGO_SETTINGS_MODULE: config.settings.pro
-      CELERY_BROKER_URL: redis://redis:6379/0
-      CELERY_RESULT_BACKEND: redis://redis:6379/0
       CELERY_LOG_LEVEL: info
     healthcheck:
       disable: true
@@ -70,19 +68,25 @@ services:
     security_opt:
       - no-new-privileges:true
 
-  rabbit-mq:
+  rabbitmq:
     image: rabbitmq:3.10-management-alpine
     restart: unless-stopped
     environment:
       RABBITMQ_DEFAULT_USER: guest
       RABBITMQ_DEFAULT_PASS: guest
+      #RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-ssl_opts.server_name_indication disable"
+    volumes:
+      - ./rabbitmq.conf:/etc/rabbitmq/rabbitmq.conf
+      - rabbitmq_data:/var/lib/rabbitmq
     healthcheck:
       test: rabbitmq-diagnostics -q ping
     deploy:
       resources:
         limits:
-          cpus: "${RABBITMQ_CPU:-2}"
-          memory: ${RABBITMQ_MEM:-512M}
+          cpus: "${RABBITMQ_CPU:-4}"
+          memory: ${RABBITMQ_MEM:-1024M}
+        reservations:
+          memory: 512M
     security_opt:
       - no-new-privileges:true
 
@@ -108,17 +112,22 @@ services:
       - ./jasmin_config/resource:/etc/jasmin/resource
       - ./jasmin_config/store:/etc/jasmin/store
       - ./jasmin_config:/etc/jasmin
+      - ./logs:/var/log/jasmin
     ports:
       - "${JASMIN_SMS_PORT:-2775}:2775"
       - "${JASMIN_DASHBOARD_PORT:-8990}:8990"
       - "${JASMIN_HTTP_API_PORT:-1401}:1401"
     depends_on:
       - redis
-      - rabbit-mq
+      - rabbitmq
     environment:
       REDIS_CLIENT_HOST: redis
-      AMQP_BROKER_HOST: rabbit-mq
+      AMQP_BROKER_HOST: rabbitmq
       AMQP_BROKER_PORT: 5672
+    sysctls:
+      - net.ipv4.tcp_keepalive_time=60
+      - net.ipv4.tcp_keepalive_intvl=10
+      - net.ipv4.tcp_keepalive_probes=5
     deploy:
       resources:
         limits:
@@ -136,21 +145,32 @@ services:
     command: bash -c "sleep 15 && pip install -U pip psycopg2-binary mysql-connector-python && exec python /build/misc/scripts/sms_logger.py"
     environment:
       DB_TYPE_MYSQL: ${DB_TYPE_MYSQL:-0}
-      AMQP_BROKER_HOST: ${AMQP_BROKER_HOST:-rabbit-mq}
+      AMQP_BROKER_HOST: ${AMQP_BROKER_HOST:-rabbitmq}
       AMQP_BROKER_PORT: ${AMQP_BROKER_PORT:-5672}
       AMQP_SPEC_FILE: '/etc/jasmin/resource/amqp0-9-1.xml'
+      AMQP_HEARTBEAT: 60  # Enable 60-second heartbeats
+      RECONNECT_DELAY: 5  # Initial reconnect delay
+      MAX_RECONNECT_DELAY: 60  # Max reconnect delay
       DB_HOST: ${DB_HOST:-db}
       DB_DATABASE: ${DB_DATABASE:-jasmin}
       DB_TABLE: ${DB_TABLE:-submit_log}
       DB_USER: ${DB_USER:-jasmin}
       DB_PASS: ${DB_PASS:-jasmin}
     depends_on:
-      - rabbit-mq
+      - rabbitmq
       - db
       - jasmin
     restart: unless-stopped
     healthcheck:
-      disable: true
+      test: ["CMD", "pgrep", "-f", "sms_logger.py"]
+      interval: 30s
+      timeout: 10s
+      retries: 3
+      start_period: 40s
+    sysctls:
+      net.ipv4.tcp_keepalive_time: 60
+      net.ipv4.tcp_keepalive_intvl: 10
+      net.ipv4.tcp_keepalive_probes: 5
 
 volumes:
   web_public:
@@ -159,7 +179,9 @@ volumes:
     driver: local
   redis_data:
     driver: local
+  rabbitmq_data:
+    driver: local
   postgres_data:
     driver: local
   monitoring_data:
-    driver: local
+    driver: local

main/core/middleware.py

@@ -1,7 +1,7 @@
 from django.conf import settings
 from django.utils.deprecation import MiddlewareMixin
 from django.core.cache import cache
-from django.db import transaction
+from django.db import transaction, connection
 from .utils import get_user_agent, get_client_ip, LazyEncoder
 from .models import ActivityLog
 
@@ -73,20 +73,54 @@ def process_request(self, request):
         self._enqueue_activity_log_creation(request, user_agent)
 
     def _enqueue_activity_log_creation(self, request, user_agent):
-        from concurrent.futures import ThreadPoolExecutor
-        with ThreadPoolExecutor(max_workers=1) as executor:
-            executor.submit(self._create_activity_log, request, user_agent)
+        # Extract data from request before threading to avoid context issues
+        log_data = {
+            'user_id': request.user.id,
+            'service': request.POST.get("s", "unknown"),
+            'method': request.method,
+            'params': self.clean_params(request.POST or request.GET or {}),
+            'path': request.path,
+            'ip': get_client_ip(request),
+            'user_agent': user_agent,
+        }
+        
+        # Use threading.Thread instead of ThreadPoolExecutor for simpler lifecycle
+        import threading
+        thread = threading.Thread(target=self._create_activity_log_safe, args=(log_data,), daemon=True)
+        thread.start()
 
+    def clean_params(self, params):
+        """Clean sensitive parameters before logging."""
+        cleaned = dict(params)
+        sensitive_keys = ['password', 'passwd', 'pwd', 'secret', 'token', 'api_key', 'apikey']
+        for key in list(cleaned.keys()):
+            if any(sensitive in key.lower() for sensitive in sensitive_keys):
+                cleaned[key] = '***REDACTED***'
+        return cleaned
+
+    def _create_activity_log_safe(self, log_data):
+        """Thread-safe activity log creation with proper DB connection handling."""
+        try:
+            self._create_activity_log(log_data)
+        except Exception as e:
+            logger.error(f"Failed to create activity log: {str(e)}", exc_info=True)
+        finally:
+            # Close database connection for this thread
+            connection.close()
+    
     @transaction.atomic
-    def _create_activity_log(self, request, user_agent):
-        params = self.clean_params(request.POST or request.GET or {})
+    def _create_activity_log(self, log_data):
+        from django.contrib.auth import get_user_model
+        User = get_user_model()
+        
+        user = User.objects.get(id=log_data['user_id'])
         activity_log = ActivityLog(
-            user=request.user,
-            service=request.POST.get("s", "unknown"),
-            method=request.method,
-            params=json.dumps(params, cls=LazyEncoder),
-            path=request.path,
-            ip=get_client_ip(request),
-            user_agent=json.dumps(user_agent.__dict__ or {}, cls=LazyEncoder),
+            user=user,
+            service=log_data['service'],
+            method=log_data['method'],
+            params=json.dumps(log_data['params'], cls=LazyEncoder),
+            path=log_data['path'],
+            ip=log_data['ip'],
+            user_agent=json.dumps(log_data['user_agent'].__dict__ or {}, cls=LazyEncoder),
         )
         activity_log.save()

main/core/tasks/__init__.py

@@ -1 +1,2 @@
 from .mail_html import mail_html_mails, mail_html_envelopes
+from .export_submit_logs import export_submit_logs_task