add celery

Author: ewdurbin

base-requirements.txt

@@ -19,6 +19,8 @@ feedparser==6.0.8
 beautifulsoup4==4.11.2
 icalendar==4.0.7
 chardet==4.0.0
+celery[redis]==5.3.6
+django-celery-beat==2.5.0
 # TODO: We may drop 'django-imagekit' completely.
 django-imagekit==4.0.2
 django-haystack==3.2.1

docker-compose.yml

@@ -14,8 +14,17 @@ services:
       test: ["CMD", "pg_isready", "-U", "pythondotorg", "-d", "pythondotorg"]
       interval: 1s
 
+  redis:
+    image: redis:7-bullseye
+    ports:
+      - "6379:6379"
+    healthcheck:
+     test: ["CMD", "redis-cli","ping"]
+     interval: 1s
+
   web:
     build: .
+    image: pythondotorg:docker-compose
     command: python manage.py runserver 0.0.0.0:8000
     volumes:
       - .:/code
@@ -27,3 +36,19 @@ services:
     depends_on:
       postgres:
         condition: service_healthy
+      redis:
+        condition: service_healthy
+
+  worker:
+    image: pythondotorg:docker-compose
+    command: celery -A pydotorg worker -B -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler
+    volumes:
+      - .:/code
+    environment:
+      DATABASE_URL: postgresql://pythondotorg:pythondotorg@postgres:5432/pythondotorg
+      DJANGO_SETTINGS_MODULE: pydotorg.settings.local
+    depends_on:
+      postgres:
+        condition: service_healthy
+      redis:
+        condition: service_healthy

pydotorg/__init__.py

@@ -0,0 +1,3 @@
+from pydotorg.celery import app as celery_app
+
+__all__ = ("celery_app",)

pydotorg/celery.py

@@ -0,0 +1,15 @@
+import os
+
+from celery import Celery
+from django.core import management
+
+os.environ.setdefault("DJANGO_SETTINGS_MODULE", "pydotorg.settings.local")
+
+app = Celery("pydotorg")
+app.config_from_object("django.conf:settings", namespace="CELERY")
+
+@app.task(bind=True)
+def run_management_command(self, command_name, args, kwargs):
+    management.call_command(command_name, *args, **kwargs)
+
+app.autodiscover_tasks()

pydotorg/settings/base.py

@@ -31,6 +31,23 @@
     )
 }
 
+# celery settings
+_REDIS_URL = config("REDIS_URL", default="redis://redis:6379/0")
+
+CELERY_BROKER_URL = _REDIS_URL
+CELERY_RESULT_BACKEND = _REDIS_URL
+
+CELERY_BEAT_SCHEDULE = {
+    # "example-management-command": {
+    #    "task": "pydotorg.celery.run_management_command",
+    #    "schedule": crontab(hour=12, minute=0),
+    #    "args": ("daily_volunteer_reminder", [], {}),
+    # },
+    # 'example-task': {
+    #     'task': 'users.tasks.example_task',
+    # },
+}
+
 ### Locale settings
 
 TIME_ZONE = 'UTC'
@@ -163,6 +180,7 @@
     'django.contrib.admin',
     'django.contrib.admindocs',
 
+    'django_celery_beat',
     'django_translation_aliases',
     'pipeline',
     'sitetree',