From d81c11b3ac08ba8d5616a94965404a9dd9103a81 Mon Sep 17 00:00:00 2001 From: "v.kaukin" Date: Sat, 27 Apr 2024 11:30:25 +0500 Subject: [PATCH 1/9] DJANGO_DRAMATIQ_TASKS_NOT_WRITES, DJANGO_DRAMATIQ_TASKS_WRITES_ONLY --- README.md | 7 +++++++ django_dramatiq/models.py | 14 +++++++++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index f2062a2..6044e1a 100644 --- a/README.md +++ b/README.md @@ -74,6 +74,13 @@ DRAMATIQ_RESULT_BACKEND = { } ``` +You can specify which actors to write to the database with "black" and "white" lists: + +``` python +DJANGO_DRAMATIQ_TASKS_NOT_WRITES = ['actor_name_that_excluded'] +DJANGO_DRAMATIQ_TASKS_WRITES_ONLY = ['actor_name_that_writes_only1', 'actor_name_that_writes_only2'] +``` + ## Usage diff --git a/django_dramatiq/models.py b/django_dramatiq/models.py index 5a2f19b..e65aea9 100644 --- a/django_dramatiq/models.py +++ b/django_dramatiq/models.py @@ -1,5 +1,7 @@ from datetime import timedelta +from typing import Optional +from django.conf import settings from django.db import models from django.utils.functional import cached_property from django.utils.timezone import now @@ -10,9 +12,19 @@ #: The database label to use when storing task metadata. DATABASE_LABEL = DjangoDramatiqConfig.tasks_database() +DJANGO_DRAMATIQ_TASKS_NOT_WRITES = getattr(settings, "DJANGO_DRAMATIQ_TASKS_NOT_WRITES", []) +DJANGO_DRAMATIQ_TASKS_WRITES_ONLY = getattr(settings, "DJANGO_DRAMATIQ_TASKS_WRITES_ONLY", []) + class TaskManager(models.Manager): - def create_or_update_from_message(self, message, **extra_fields): + def create_or_update_from_message(self, message, **extra_fields) -> Optional['Task']: + + # black and write lists + if DJANGO_DRAMATIQ_TASKS_WRITES_ONLY and message.actor_name not in DJANGO_DRAMATIQ_TASKS_WRITES_ONLY: + return None + if DJANGO_DRAMATIQ_TASKS_NOT_WRITES and message.actor_name in DJANGO_DRAMATIQ_TASKS_NOT_WRITES: + return None + task, _ = self.using(DATABASE_LABEL).update_or_create( id=message.message_id, defaults={ From 69e7766e9b1eefa511626e91b9576a2410373abb Mon Sep 17 00:00:00 2001 From: "v.kaukin" Date: Thu, 30 Jan 2025 09:19:42 +0500 Subject: [PATCH 2/9] resolve --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6044e1a..b89aa0f 100644 --- a/README.md +++ b/README.md @@ -82,7 +82,7 @@ DJANGO_DRAMATIQ_TASKS_WRITES_ONLY = ['actor_name_that_writes_only1', 'actor_name ``` -## Usage +## Getting Started ### Declaring tasks From f50de6c66e1c0108e553b35acbfb47961aefd07e Mon Sep 17 00:00:00 2001 From: "v.kaukin" Date: Thu, 30 Jan 2025 09:24:27 +0500 Subject: [PATCH 3/9] resolve --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index b89aa0f..3523a93 100644 --- a/README.md +++ b/README.md @@ -74,14 +74,13 @@ DRAMATIQ_RESULT_BACKEND = { } ``` -You can specify which actors to write to the database with "black" and "white" lists: +You can specify which actors to write to the database: ``` python DJANGO_DRAMATIQ_TASKS_NOT_WRITES = ['actor_name_that_excluded'] DJANGO_DRAMATIQ_TASKS_WRITES_ONLY = ['actor_name_that_writes_only1', 'actor_name_that_writes_only2'] ``` - ## Getting Started ### Declaring tasks From 2e6a63e660f340d5bc75fd62bdc8cec7b641ac49 Mon Sep 17 00:00:00 2001 From: "v.kaukin" Date: Thu, 30 Jan 2025 09:28:44 +0500 Subject: [PATCH 4/9] resolve --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3523a93..5780f8c 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ DJANGO_DRAMATIQ_TASKS_NOT_WRITES = ['actor_name_that_excluded'] DJANGO_DRAMATIQ_TASKS_WRITES_ONLY = ['actor_name_that_writes_only1', 'actor_name_that_writes_only2'] ``` -## Getting Started +## Usage ### Declaring tasks From 826b14f4dcfc3c6337a1cbdbf2696b66d6d7d8b1 Mon Sep 17 00:00:00 2001 From: "v.kaukin" Date: Thu, 30 Jan 2025 10:16:53 +0500 Subject: [PATCH 5/9] rename, resolve, doc --- README.md | 128 +++++++++++++++++++++----------------- django_dramatiq/models.py | 9 ++- 2 files changed, 76 insertions(+), 61 deletions(-) diff --git a/README.md b/README.md index 5780f8c..1af7393 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,42 @@ -# django_dramatiq +# Django Dramatiq -[![Build Status](https://github.com/Bogdanp/django_dramatiq/actions/workflows/push.yml/badge.svg)](https://github.com/Bogdanp/django_dramatiq/actions/workflows/push.yml) +![Python Version](https://img.shields.io/pypi/pyversions/django-dramatiq) +![Django Versions](https://img.shields.io/pypi/frameworkversions/django/django-dramatiq) +[![Build Status](https://github.com/Bogdanp/django_dramatiq/actions/workflows/ci.yml/badge.svg)](https://github.com/Bogdanp/django_dramatiq/actions/workflows/ci.yml) [![PyPI version](https://badge.fury.io/py/django-dramatiq.svg)](https://badge.fury.io/py/django-dramatiq) +[![License](https://img.shields.io/badge/License-Apache_2.0-orange.svg)](https://opensource.org/licenses/Apache-2.0) -**django_dramatiq** is a Django app that integrates with [Dramatiq][dramatiq]. +Seemlessly integrate [Dramatiq][dramatiq] with your Django project! +# Contents -## Requirements +- [Installation](#installation) +- [Getting Started](#getting-started) +- [Testing](#testing) +- [Middleware](#middleware) +- [Advanced Usage](#advanced-usage) +- [Third-Party Support](#third-party-support) +- [Example App](#example) -* [Django][django] 1.11+ -* [Dramatiq][dramatiq] 0.18+ +## Installation +To install, ensure both Django Dramtiq and Dramatiq are installed, along with RabbitMQ: -## Example + pip install django-dramatiq 'dramatiq[rabbitmq]' -You can find an example application built with django_dramatiq [here][example]. +Or with Redis: + pip install django-dramatiq 'dramatiq[redis]' -## Installation +If you would like to install with `watch`: + + pip install django-dramatiq 'dramatiq[rabbitmq, watch]' - pip install django-dramatiq Add `django_dramatiq` to installed apps *before* any of your custom apps: ``` python -import os - INSTALLED_APPS = [ "django_dramatiq", @@ -40,7 +50,7 @@ Configure your broker in `settings.py`: ``` python DRAMATIQ_BROKER = { - "BROKER": "dramatiq.brokers.rabbitmq.RabbitmqBroker", + "BROKER": "dramatiq.brokers.rabbitmq.RabbitmqBroker", "OPTIONS": { "url": "amqp://localhost:5672", }, @@ -60,32 +70,11 @@ DRAMATIQ_BROKER = { DRAMATIQ_TASKS_DATABASE = "default" ``` -You may also configure a result backend: - -``` python -DRAMATIQ_RESULT_BACKEND = { - "BACKEND": "dramatiq.results.backends.redis.RedisBackend", - "BACKEND_OPTIONS": { - "url": "redis://localhost:6379", - }, - "MIDDLEWARE_OPTIONS": { - "result_ttl": 60000 - } -} -``` - -You can specify which actors to write to the database: - -``` python -DJANGO_DRAMATIQ_TASKS_NOT_WRITES = ['actor_name_that_excluded'] -DJANGO_DRAMATIQ_TASKS_WRITES_ONLY = ['actor_name_that_writes_only1', 'actor_name_that_writes_only2'] -``` - -## Usage +## Getting Started ### Declaring tasks -django_dramatiq will auto-discover tasks defined in `tasks` modules in +Django Dramatiq will auto-discover tasks defined in `tasks` modules in each of your installed apps. For example, if you have an app named `customers`, your tasks for that app should live in a module called `customers.tasks`: @@ -112,7 +101,7 @@ DRAMATIQ_AUTODISCOVER_MODULES = ["tasks", "services"] ### Running workers -django_dramatiq comes with a management command you can use to +Django Dramatiq comes with a management command you can use to auto-discover task modules and run workers: python manage.py rundramatiq @@ -134,7 +123,32 @@ The wildcard detection will ignore all sub modules from that point on. You will need to ignore the module itself if you don't want the `__init__.py` to be processed. -### Testing +### Results Backend + +You may also configure a result backend: + +``` python +DRAMATIQ_RESULT_BACKEND = { + "BACKEND": "dramatiq.results.backends.redis.RedisBackend", + "BACKEND_OPTIONS": { + "url": "redis://localhost:6379", + }, + "MIDDLEWARE_OPTIONS": { + "result_ttl": 1000 * 60 * 10 + } +} +``` + +### Actors to write to the database + +You can specify which actors are allowed or blocked from writing to the database using the following settings: + +``` python +DJANGO_DRAMATIQ_TASKS_BLOCKLIST = ['actor_name_that_excluded'] +DJANGO_DRAMATIQ_TASKS_ALLOWLIST = ['actor_name_that_writes_only1', 'actor_name_that_writes_only2'] +``` + +## Testing You should have a separate settings file for test. In that file, overwrite the broker to use Dramatiq's [StubBroker][stubbroker]: @@ -195,6 +209,12 @@ def test_customers_can_be_emailed(transactional_db, broker, worker, mailoutbox): assert mailoutbox[0].subject == "Welcome Jim!" ``` + +> [!NOTE] +> If your tests rely on the results of the actor, you may experience inconsistent results. Due to the nature of the worker and test running in seperate threads, the test DB state may be different. +> +> To solve this you need to add the addtional `@pytest.mark.django_db(transaction=True)` decorator. + #### Using unittest A simple test case has been provided that will automatically set up the @@ -225,18 +245,23 @@ class CustomerTestCase(DramatiqTestCase): self.assertEqual(mail.outbox[0].subject, "Welcome Jim!") ``` -#### Cleaning up old tasks + +## Advanced Usage + +### Cleaning up old tasks The `AdminMiddleware` stores task metadata in a relational DB so it's a good idea to garbage collect that data every once in a while. You can use the `delete_old_tasks` actor to achieve this on a cron: ``` python -delete_old_tasks.send(max_task_age=86400) +from django_dramatiq.tasks import delete_old_tasks + +delete_old_tasks.send(max_task_age=60 * 60 * 24) ``` -### Middleware +## Middleware
django_dramatiq.middleware.DbConnectionsMiddleware
@@ -252,7 +277,7 @@ delete_old_tasks.send(max_task_age=86400)
-#### Custom keyword arguments to Middleware +### Custom keyword arguments to Middleware Some middleware classes require dynamic arguments. An example of this would be the backend argument to `dramatiq.middleware.GroupCallbacks`. @@ -282,9 +307,6 @@ class CustomDjangoDramatiqConfig(DjangoDramatiqConfig): @classmethod def middleware_groupcallbacks_kwargs(cls): return {"rate_limiter_backend": cls.get_rate_limiter_backend()} - - -CustomDjangoDramatiqConfig.initialize() ``` Notice the naming convention, to provide arguments to @@ -304,7 +326,9 @@ INSTALLED_APPS = [ ``` -### Usage with [django-configurations] +## Third-Party Support + +#### Usage with [django-configurations] To use django_dramatiq together with [django-configurations] you need to define your own `rundramatiq` command as a subclass of the one in @@ -334,20 +358,12 @@ install(check_options=True) django.setup() ``` -## Running project tests locally - -Install the dev dependencies with `pip install -e '.[dev]'` and then run `tox`. - +## Example -## License +You can find an example application built with Django Dramatiq [here](/examples/basic/README.md). -django_dramatiq is licensed under Apache 2.0. Please see -[LICENSE][license] for licensing details. -[django]: http://djangoproject.com/ [dramatiq]: https://github.com/Bogdanp/dramatiq -[example]: https://github.com/Bogdanp/django_dramatiq_example -[license]: https://github.com/Bogdanp/django_dramatiq/blob/master/LICENSE [pytest-django]: https://pytest-django.readthedocs.io/en/latest/index.html [stubbroker]: https://dramatiq.io/reference.html#dramatiq.brokers.stub.StubBroker [django-configurations]: https://github.com/jazzband/django-configurations/ diff --git a/django_dramatiq/models.py b/django_dramatiq/models.py index e65aea9..7cea4e0 100644 --- a/django_dramatiq/models.py +++ b/django_dramatiq/models.py @@ -12,17 +12,16 @@ #: The database label to use when storing task metadata. DATABASE_LABEL = DjangoDramatiqConfig.tasks_database() -DJANGO_DRAMATIQ_TASKS_NOT_WRITES = getattr(settings, "DJANGO_DRAMATIQ_TASKS_NOT_WRITES", []) -DJANGO_DRAMATIQ_TASKS_WRITES_ONLY = getattr(settings, "DJANGO_DRAMATIQ_TASKS_WRITES_ONLY", []) +DJANGO_DRAMATIQ_TASKS_BLOCKLIST = getattr(settings, "DJANGO_DRAMATIQ_TASKS_BLOCKLIST", []) +DJANGO_DRAMATIQ_TASKS_ALLOWLIST = getattr(settings, "DJANGO_DRAMATIQ_TASKS_ALLOWLIST", []) class TaskManager(models.Manager): def create_or_update_from_message(self, message, **extra_fields) -> Optional['Task']: - # black and write lists - if DJANGO_DRAMATIQ_TASKS_WRITES_ONLY and message.actor_name not in DJANGO_DRAMATIQ_TASKS_WRITES_ONLY: + if DJANGO_DRAMATIQ_TASKS_ALLOWLIST and message.actor_name not in DJANGO_DRAMATIQ_TASKS_ALLOWLIST: return None - if DJANGO_DRAMATIQ_TASKS_NOT_WRITES and message.actor_name in DJANGO_DRAMATIQ_TASKS_NOT_WRITES: + if DJANGO_DRAMATIQ_TASKS_BLOCKLIST and message.actor_name in DJANGO_DRAMATIQ_TASKS_BLOCKLIST: return None task, _ = self.using(DATABASE_LABEL).update_or_create( From 1b84b15db7033961b300a43729c3a7ad47665b17 Mon Sep 17 00:00:00 2001 From: "v.kaukin" Date: Thu, 30 Jan 2025 10:33:49 +0500 Subject: [PATCH 6/9] resolve... --- README.md | 113 ++++++++++++++++++++++++------------------------------ 1 file changed, 50 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 1af7393..7311f92 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,32 @@ -# Django Dramatiq +# django_dramatiq -![Python Version](https://img.shields.io/pypi/pyversions/django-dramatiq) -![Django Versions](https://img.shields.io/pypi/frameworkversions/django/django-dramatiq) -[![Build Status](https://github.com/Bogdanp/django_dramatiq/actions/workflows/ci.yml/badge.svg)](https://github.com/Bogdanp/django_dramatiq/actions/workflows/ci.yml) +[![Build Status](https://github.com/Bogdanp/django_dramatiq/actions/workflows/push.yml/badge.svg)](https://github.com/Bogdanp/django_dramatiq/actions/workflows/push.yml) [![PyPI version](https://badge.fury.io/py/django-dramatiq.svg)](https://badge.fury.io/py/django-dramatiq) -[![License](https://img.shields.io/badge/License-Apache_2.0-orange.svg)](https://opensource.org/licenses/Apache-2.0) -Seemlessly integrate [Dramatiq][dramatiq] with your Django project! +**django_dramatiq** is a Django app that integrates with [Dramatiq][dramatiq]. -# Contents -- [Installation](#installation) -- [Getting Started](#getting-started) -- [Testing](#testing) -- [Middleware](#middleware) -- [Advanced Usage](#advanced-usage) -- [Third-Party Support](#third-party-support) -- [Example App](#example) +## Requirements -## Installation - -To install, ensure both Django Dramtiq and Dramatiq are installed, along with RabbitMQ: +* [Django][django] 1.11+ +* [Dramatiq][dramatiq] 0.18+ - pip install django-dramatiq 'dramatiq[rabbitmq]' -Or with Redis: +## Example - pip install django-dramatiq 'dramatiq[redis]' +You can find an example application built with django_dramatiq [here][example]. -If you would like to install with `watch`: - pip install django-dramatiq 'dramatiq[rabbitmq, watch]' +## Installation + pip install django-dramatiq Add `django_dramatiq` to installed apps *before* any of your custom apps: ``` python +import os + INSTALLED_APPS = [ "django_dramatiq", @@ -50,7 +40,7 @@ Configure your broker in `settings.py`: ``` python DRAMATIQ_BROKER = { - "BROKER": "dramatiq.brokers.rabbitmq.RabbitmqBroker", + "BROKER": "dramatiq.brokers.rabbitmq.RabbitmqBroker", "OPTIONS": { "url": "amqp://localhost:5672", }, @@ -70,11 +60,26 @@ DRAMATIQ_BROKER = { DRAMATIQ_TASKS_DATABASE = "default" ``` -## Getting Started +You may also configure a result backend: + +``` python +DRAMATIQ_RESULT_BACKEND = { + "BACKEND": "dramatiq.results.backends.redis.RedisBackend", + "BACKEND_OPTIONS": { + "url": "redis://localhost:6379", + }, + "MIDDLEWARE_OPTIONS": { + "result_ttl": 60000 + } +} +``` + + +## Usage ### Declaring tasks -Django Dramatiq will auto-discover tasks defined in `tasks` modules in +django_dramatiq will auto-discover tasks defined in `tasks` modules in each of your installed apps. For example, if you have an app named `customers`, your tasks for that app should live in a module called `customers.tasks`: @@ -101,7 +106,7 @@ DRAMATIQ_AUTODISCOVER_MODULES = ["tasks", "services"] ### Running workers -Django Dramatiq comes with a management command you can use to +django_dramatiq comes with a management command you can use to auto-discover task modules and run workers: python manage.py rundramatiq @@ -123,22 +128,6 @@ The wildcard detection will ignore all sub modules from that point on. You will need to ignore the module itself if you don't want the `__init__.py` to be processed. -### Results Backend - -You may also configure a result backend: - -``` python -DRAMATIQ_RESULT_BACKEND = { - "BACKEND": "dramatiq.results.backends.redis.RedisBackend", - "BACKEND_OPTIONS": { - "url": "redis://localhost:6379", - }, - "MIDDLEWARE_OPTIONS": { - "result_ttl": 1000 * 60 * 10 - } -} -``` - ### Actors to write to the database You can specify which actors are allowed or blocked from writing to the database using the following settings: @@ -148,7 +137,7 @@ DJANGO_DRAMATIQ_TASKS_BLOCKLIST = ['actor_name_that_excluded'] DJANGO_DRAMATIQ_TASKS_ALLOWLIST = ['actor_name_that_writes_only1', 'actor_name_that_writes_only2'] ``` -## Testing +### Testing You should have a separate settings file for test. In that file, overwrite the broker to use Dramatiq's [StubBroker][stubbroker]: @@ -209,12 +198,6 @@ def test_customers_can_be_emailed(transactional_db, broker, worker, mailoutbox): assert mailoutbox[0].subject == "Welcome Jim!" ``` - -> [!NOTE] -> If your tests rely on the results of the actor, you may experience inconsistent results. Due to the nature of the worker and test running in seperate threads, the test DB state may be different. -> -> To solve this you need to add the addtional `@pytest.mark.django_db(transaction=True)` decorator. - #### Using unittest A simple test case has been provided that will automatically set up the @@ -245,23 +228,18 @@ class CustomerTestCase(DramatiqTestCase): self.assertEqual(mail.outbox[0].subject, "Welcome Jim!") ``` - -## Advanced Usage - -### Cleaning up old tasks +#### Cleaning up old tasks The `AdminMiddleware` stores task metadata in a relational DB so it's a good idea to garbage collect that data every once in a while. You can use the `delete_old_tasks` actor to achieve this on a cron: ``` python -from django_dramatiq.tasks import delete_old_tasks - -delete_old_tasks.send(max_task_age=60 * 60 * 24) +delete_old_tasks.send(max_task_age=86400) ``` -## Middleware +### Middleware
django_dramatiq.middleware.DbConnectionsMiddleware
@@ -277,7 +255,7 @@ delete_old_tasks.send(max_task_age=60 * 60 * 24)
-### Custom keyword arguments to Middleware +#### Custom keyword arguments to Middleware Some middleware classes require dynamic arguments. An example of this would be the backend argument to `dramatiq.middleware.GroupCallbacks`. @@ -307,6 +285,9 @@ class CustomDjangoDramatiqConfig(DjangoDramatiqConfig): @classmethod def middleware_groupcallbacks_kwargs(cls): return {"rate_limiter_backend": cls.get_rate_limiter_backend()} + + +CustomDjangoDramatiqConfig.initialize() ``` Notice the naming convention, to provide arguments to @@ -326,9 +307,7 @@ INSTALLED_APPS = [ ``` -## Third-Party Support - -#### Usage with [django-configurations] +### Usage with [django-configurations] To use django_dramatiq together with [django-configurations] you need to define your own `rundramatiq` command as a subclass of the one in @@ -358,12 +337,20 @@ install(check_options=True) django.setup() ``` -## Example +## Running project tests locally + +Install the dev dependencies with `pip install -e '.[dev]'` and then run `tox`. + -You can find an example application built with Django Dramatiq [here](/examples/basic/README.md). +## License +django_dramatiq is licensed under Apache 2.0. Please see +[LICENSE][license] for licensing details. +[django]: http://djangoproject.com/ [dramatiq]: https://github.com/Bogdanp/dramatiq +[example]: https://github.com/Bogdanp/django_dramatiq_example +[license]: https://github.com/Bogdanp/django_dramatiq/blob/master/LICENSE [pytest-django]: https://pytest-django.readthedocs.io/en/latest/index.html [stubbroker]: https://dramatiq.io/reference.html#dramatiq.brokers.stub.StubBroker [django-configurations]: https://github.com/jazzband/django-configurations/ From 320dfc30b9fafc5a7a248ef18447225de028f2dd Mon Sep 17 00:00:00 2001 From: "v.kaukin" Date: Thu, 30 Jan 2025 10:36:48 +0500 Subject: [PATCH 7/9] resolve... --- README.md | 113 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 63 insertions(+), 50 deletions(-) diff --git a/README.md b/README.md index 7311f92..1af7393 100644 --- a/README.md +++ b/README.md @@ -1,32 +1,42 @@ -# django_dramatiq +# Django Dramatiq -[![Build Status](https://github.com/Bogdanp/django_dramatiq/actions/workflows/push.yml/badge.svg)](https://github.com/Bogdanp/django_dramatiq/actions/workflows/push.yml) +![Python Version](https://img.shields.io/pypi/pyversions/django-dramatiq) +![Django Versions](https://img.shields.io/pypi/frameworkversions/django/django-dramatiq) +[![Build Status](https://github.com/Bogdanp/django_dramatiq/actions/workflows/ci.yml/badge.svg)](https://github.com/Bogdanp/django_dramatiq/actions/workflows/ci.yml) [![PyPI version](https://badge.fury.io/py/django-dramatiq.svg)](https://badge.fury.io/py/django-dramatiq) +[![License](https://img.shields.io/badge/License-Apache_2.0-orange.svg)](https://opensource.org/licenses/Apache-2.0) -**django_dramatiq** is a Django app that integrates with [Dramatiq][dramatiq]. +Seemlessly integrate [Dramatiq][dramatiq] with your Django project! +# Contents -## Requirements +- [Installation](#installation) +- [Getting Started](#getting-started) +- [Testing](#testing) +- [Middleware](#middleware) +- [Advanced Usage](#advanced-usage) +- [Third-Party Support](#third-party-support) +- [Example App](#example) -* [Django][django] 1.11+ -* [Dramatiq][dramatiq] 0.18+ +## Installation +To install, ensure both Django Dramtiq and Dramatiq are installed, along with RabbitMQ: -## Example + pip install django-dramatiq 'dramatiq[rabbitmq]' -You can find an example application built with django_dramatiq [here][example]. +Or with Redis: + pip install django-dramatiq 'dramatiq[redis]' -## Installation +If you would like to install with `watch`: + + pip install django-dramatiq 'dramatiq[rabbitmq, watch]' - pip install django-dramatiq Add `django_dramatiq` to installed apps *before* any of your custom apps: ``` python -import os - INSTALLED_APPS = [ "django_dramatiq", @@ -40,7 +50,7 @@ Configure your broker in `settings.py`: ``` python DRAMATIQ_BROKER = { - "BROKER": "dramatiq.brokers.rabbitmq.RabbitmqBroker", + "BROKER": "dramatiq.brokers.rabbitmq.RabbitmqBroker", "OPTIONS": { "url": "amqp://localhost:5672", }, @@ -60,26 +70,11 @@ DRAMATIQ_BROKER = { DRAMATIQ_TASKS_DATABASE = "default" ``` -You may also configure a result backend: - -``` python -DRAMATIQ_RESULT_BACKEND = { - "BACKEND": "dramatiq.results.backends.redis.RedisBackend", - "BACKEND_OPTIONS": { - "url": "redis://localhost:6379", - }, - "MIDDLEWARE_OPTIONS": { - "result_ttl": 60000 - } -} -``` - - -## Usage +## Getting Started ### Declaring tasks -django_dramatiq will auto-discover tasks defined in `tasks` modules in +Django Dramatiq will auto-discover tasks defined in `tasks` modules in each of your installed apps. For example, if you have an app named `customers`, your tasks for that app should live in a module called `customers.tasks`: @@ -106,7 +101,7 @@ DRAMATIQ_AUTODISCOVER_MODULES = ["tasks", "services"] ### Running workers -django_dramatiq comes with a management command you can use to +Django Dramatiq comes with a management command you can use to auto-discover task modules and run workers: python manage.py rundramatiq @@ -128,6 +123,22 @@ The wildcard detection will ignore all sub modules from that point on. You will need to ignore the module itself if you don't want the `__init__.py` to be processed. +### Results Backend + +You may also configure a result backend: + +``` python +DRAMATIQ_RESULT_BACKEND = { + "BACKEND": "dramatiq.results.backends.redis.RedisBackend", + "BACKEND_OPTIONS": { + "url": "redis://localhost:6379", + }, + "MIDDLEWARE_OPTIONS": { + "result_ttl": 1000 * 60 * 10 + } +} +``` + ### Actors to write to the database You can specify which actors are allowed or blocked from writing to the database using the following settings: @@ -137,7 +148,7 @@ DJANGO_DRAMATIQ_TASKS_BLOCKLIST = ['actor_name_that_excluded'] DJANGO_DRAMATIQ_TASKS_ALLOWLIST = ['actor_name_that_writes_only1', 'actor_name_that_writes_only2'] ``` -### Testing +## Testing You should have a separate settings file for test. In that file, overwrite the broker to use Dramatiq's [StubBroker][stubbroker]: @@ -198,6 +209,12 @@ def test_customers_can_be_emailed(transactional_db, broker, worker, mailoutbox): assert mailoutbox[0].subject == "Welcome Jim!" ``` + +> [!NOTE] +> If your tests rely on the results of the actor, you may experience inconsistent results. Due to the nature of the worker and test running in seperate threads, the test DB state may be different. +> +> To solve this you need to add the addtional `@pytest.mark.django_db(transaction=True)` decorator. + #### Using unittest A simple test case has been provided that will automatically set up the @@ -228,18 +245,23 @@ class CustomerTestCase(DramatiqTestCase): self.assertEqual(mail.outbox[0].subject, "Welcome Jim!") ``` -#### Cleaning up old tasks + +## Advanced Usage + +### Cleaning up old tasks The `AdminMiddleware` stores task metadata in a relational DB so it's a good idea to garbage collect that data every once in a while. You can use the `delete_old_tasks` actor to achieve this on a cron: ``` python -delete_old_tasks.send(max_task_age=86400) +from django_dramatiq.tasks import delete_old_tasks + +delete_old_tasks.send(max_task_age=60 * 60 * 24) ``` -### Middleware +## Middleware
django_dramatiq.middleware.DbConnectionsMiddleware
@@ -255,7 +277,7 @@ delete_old_tasks.send(max_task_age=86400)
-#### Custom keyword arguments to Middleware +### Custom keyword arguments to Middleware Some middleware classes require dynamic arguments. An example of this would be the backend argument to `dramatiq.middleware.GroupCallbacks`. @@ -285,9 +307,6 @@ class CustomDjangoDramatiqConfig(DjangoDramatiqConfig): @classmethod def middleware_groupcallbacks_kwargs(cls): return {"rate_limiter_backend": cls.get_rate_limiter_backend()} - - -CustomDjangoDramatiqConfig.initialize() ``` Notice the naming convention, to provide arguments to @@ -307,7 +326,9 @@ INSTALLED_APPS = [ ``` -### Usage with [django-configurations] +## Third-Party Support + +#### Usage with [django-configurations] To use django_dramatiq together with [django-configurations] you need to define your own `rundramatiq` command as a subclass of the one in @@ -337,20 +358,12 @@ install(check_options=True) django.setup() ``` -## Running project tests locally - -Install the dev dependencies with `pip install -e '.[dev]'` and then run `tox`. - +## Example -## License +You can find an example application built with Django Dramatiq [here](/examples/basic/README.md). -django_dramatiq is licensed under Apache 2.0. Please see -[LICENSE][license] for licensing details. -[django]: http://djangoproject.com/ [dramatiq]: https://github.com/Bogdanp/dramatiq -[example]: https://github.com/Bogdanp/django_dramatiq_example -[license]: https://github.com/Bogdanp/django_dramatiq/blob/master/LICENSE [pytest-django]: https://pytest-django.readthedocs.io/en/latest/index.html [stubbroker]: https://dramatiq.io/reference.html#dramatiq.brokers.stub.StubBroker [django-configurations]: https://github.com/jazzband/django-configurations/ From f7326ad512a935714f855d7c8c6f6bf630a16881 Mon Sep 17 00:00:00 2001 From: "v.kaukin" Date: Thu, 30 Jan 2025 10:50:47 +0500 Subject: [PATCH 8/9] resolve... --- README.md | 113 ++++++++++++++++++++++++------------------------------ 1 file changed, 50 insertions(+), 63 deletions(-) diff --git a/README.md b/README.md index 1af7393..7311f92 100644 --- a/README.md +++ b/README.md @@ -1,42 +1,32 @@ -# Django Dramatiq +# django_dramatiq -![Python Version](https://img.shields.io/pypi/pyversions/django-dramatiq) -![Django Versions](https://img.shields.io/pypi/frameworkversions/django/django-dramatiq) -[![Build Status](https://github.com/Bogdanp/django_dramatiq/actions/workflows/ci.yml/badge.svg)](https://github.com/Bogdanp/django_dramatiq/actions/workflows/ci.yml) +[![Build Status](https://github.com/Bogdanp/django_dramatiq/actions/workflows/push.yml/badge.svg)](https://github.com/Bogdanp/django_dramatiq/actions/workflows/push.yml) [![PyPI version](https://badge.fury.io/py/django-dramatiq.svg)](https://badge.fury.io/py/django-dramatiq) -[![License](https://img.shields.io/badge/License-Apache_2.0-orange.svg)](https://opensource.org/licenses/Apache-2.0) -Seemlessly integrate [Dramatiq][dramatiq] with your Django project! +**django_dramatiq** is a Django app that integrates with [Dramatiq][dramatiq]. -# Contents -- [Installation](#installation) -- [Getting Started](#getting-started) -- [Testing](#testing) -- [Middleware](#middleware) -- [Advanced Usage](#advanced-usage) -- [Third-Party Support](#third-party-support) -- [Example App](#example) +## Requirements -## Installation - -To install, ensure both Django Dramtiq and Dramatiq are installed, along with RabbitMQ: +* [Django][django] 1.11+ +* [Dramatiq][dramatiq] 0.18+ - pip install django-dramatiq 'dramatiq[rabbitmq]' -Or with Redis: +## Example - pip install django-dramatiq 'dramatiq[redis]' +You can find an example application built with django_dramatiq [here][example]. -If you would like to install with `watch`: - pip install django-dramatiq 'dramatiq[rabbitmq, watch]' +## Installation + pip install django-dramatiq Add `django_dramatiq` to installed apps *before* any of your custom apps: ``` python +import os + INSTALLED_APPS = [ "django_dramatiq", @@ -50,7 +40,7 @@ Configure your broker in `settings.py`: ``` python DRAMATIQ_BROKER = { - "BROKER": "dramatiq.brokers.rabbitmq.RabbitmqBroker", + "BROKER": "dramatiq.brokers.rabbitmq.RabbitmqBroker", "OPTIONS": { "url": "amqp://localhost:5672", }, @@ -70,11 +60,26 @@ DRAMATIQ_BROKER = { DRAMATIQ_TASKS_DATABASE = "default" ``` -## Getting Started +You may also configure a result backend: + +``` python +DRAMATIQ_RESULT_BACKEND = { + "BACKEND": "dramatiq.results.backends.redis.RedisBackend", + "BACKEND_OPTIONS": { + "url": "redis://localhost:6379", + }, + "MIDDLEWARE_OPTIONS": { + "result_ttl": 60000 + } +} +``` + + +## Usage ### Declaring tasks -Django Dramatiq will auto-discover tasks defined in `tasks` modules in +django_dramatiq will auto-discover tasks defined in `tasks` modules in each of your installed apps. For example, if you have an app named `customers`, your tasks for that app should live in a module called `customers.tasks`: @@ -101,7 +106,7 @@ DRAMATIQ_AUTODISCOVER_MODULES = ["tasks", "services"] ### Running workers -Django Dramatiq comes with a management command you can use to +django_dramatiq comes with a management command you can use to auto-discover task modules and run workers: python manage.py rundramatiq @@ -123,22 +128,6 @@ The wildcard detection will ignore all sub modules from that point on. You will need to ignore the module itself if you don't want the `__init__.py` to be processed. -### Results Backend - -You may also configure a result backend: - -``` python -DRAMATIQ_RESULT_BACKEND = { - "BACKEND": "dramatiq.results.backends.redis.RedisBackend", - "BACKEND_OPTIONS": { - "url": "redis://localhost:6379", - }, - "MIDDLEWARE_OPTIONS": { - "result_ttl": 1000 * 60 * 10 - } -} -``` - ### Actors to write to the database You can specify which actors are allowed or blocked from writing to the database using the following settings: @@ -148,7 +137,7 @@ DJANGO_DRAMATIQ_TASKS_BLOCKLIST = ['actor_name_that_excluded'] DJANGO_DRAMATIQ_TASKS_ALLOWLIST = ['actor_name_that_writes_only1', 'actor_name_that_writes_only2'] ``` -## Testing +### Testing You should have a separate settings file for test. In that file, overwrite the broker to use Dramatiq's [StubBroker][stubbroker]: @@ -209,12 +198,6 @@ def test_customers_can_be_emailed(transactional_db, broker, worker, mailoutbox): assert mailoutbox[0].subject == "Welcome Jim!" ``` - -> [!NOTE] -> If your tests rely on the results of the actor, you may experience inconsistent results. Due to the nature of the worker and test running in seperate threads, the test DB state may be different. -> -> To solve this you need to add the addtional `@pytest.mark.django_db(transaction=True)` decorator. - #### Using unittest A simple test case has been provided that will automatically set up the @@ -245,23 +228,18 @@ class CustomerTestCase(DramatiqTestCase): self.assertEqual(mail.outbox[0].subject, "Welcome Jim!") ``` - -## Advanced Usage - -### Cleaning up old tasks +#### Cleaning up old tasks The `AdminMiddleware` stores task metadata in a relational DB so it's a good idea to garbage collect that data every once in a while. You can use the `delete_old_tasks` actor to achieve this on a cron: ``` python -from django_dramatiq.tasks import delete_old_tasks - -delete_old_tasks.send(max_task_age=60 * 60 * 24) +delete_old_tasks.send(max_task_age=86400) ``` -## Middleware +### Middleware
django_dramatiq.middleware.DbConnectionsMiddleware
@@ -277,7 +255,7 @@ delete_old_tasks.send(max_task_age=60 * 60 * 24)
-### Custom keyword arguments to Middleware +#### Custom keyword arguments to Middleware Some middleware classes require dynamic arguments. An example of this would be the backend argument to `dramatiq.middleware.GroupCallbacks`. @@ -307,6 +285,9 @@ class CustomDjangoDramatiqConfig(DjangoDramatiqConfig): @classmethod def middleware_groupcallbacks_kwargs(cls): return {"rate_limiter_backend": cls.get_rate_limiter_backend()} + + +CustomDjangoDramatiqConfig.initialize() ``` Notice the naming convention, to provide arguments to @@ -326,9 +307,7 @@ INSTALLED_APPS = [ ``` -## Third-Party Support - -#### Usage with [django-configurations] +### Usage with [django-configurations] To use django_dramatiq together with [django-configurations] you need to define your own `rundramatiq` command as a subclass of the one in @@ -358,12 +337,20 @@ install(check_options=True) django.setup() ``` -## Example +## Running project tests locally + +Install the dev dependencies with `pip install -e '.[dev]'` and then run `tox`. + -You can find an example application built with Django Dramatiq [here](/examples/basic/README.md). +## License +django_dramatiq is licensed under Apache 2.0. Please see +[LICENSE][license] for licensing details. +[django]: http://djangoproject.com/ [dramatiq]: https://github.com/Bogdanp/dramatiq +[example]: https://github.com/Bogdanp/django_dramatiq_example +[license]: https://github.com/Bogdanp/django_dramatiq/blob/master/LICENSE [pytest-django]: https://pytest-django.readthedocs.io/en/latest/index.html [stubbroker]: https://dramatiq.io/reference.html#dramatiq.brokers.stub.StubBroker [django-configurations]: https://github.com/jazzband/django-configurations/ From 649722ef286669e341f98080c629dc5304b5658d Mon Sep 17 00:00:00 2001 From: "v.kaukin" Date: Fri, 14 Feb 2025 14:21:24 +0500 Subject: [PATCH 9/9] del res type for create_or_update_from_message --- django_dramatiq/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/django_dramatiq/models.py b/django_dramatiq/models.py index 7cea4e0..4150abd 100644 --- a/django_dramatiq/models.py +++ b/django_dramatiq/models.py @@ -17,7 +17,7 @@ class TaskManager(models.Manager): - def create_or_update_from_message(self, message, **extra_fields) -> Optional['Task']: + def create_or_update_from_message(self, message, **extra_fields): if DJANGO_DRAMATIQ_TASKS_ALLOWLIST and message.actor_name not in DJANGO_DRAMATIQ_TASKS_ALLOWLIST: return None