Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
37 changes: 20 additions & 17 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -14,17 +14,17 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v7

- uses: "actions/setup-python@v3"
- uses: actions/setup-python@v7
with:
python-version: "3.10"

- name: "Install dependencies"
run: python -m pip install -r requirements/pkgutils.txt

- name: "Run pyupgrade"
run: pyupgrade --py37-plus **/*.py
run: pyupgrade --py310-plus **/*.py

- name: "Run flake8"
run: flake8
Expand All @@ -43,9 +43,9 @@ jobs:
runs-on: ubuntu-latest

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v7

- uses: "actions/setup-python@v3"
- uses: actions/setup-python@v7
with:
python-version: "3.10"

Expand All @@ -69,9 +69,9 @@ jobs:
python-version: ["3.10", "3.11", "3.12", "3.13", "3.14", "pypy-3.10", "pypy-3.11"]

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v7

- uses: "actions/setup-python@v2"
- uses: actions/setup-python@v7
with:
python-version: "${{ matrix.python-version }}"

Expand All @@ -81,7 +81,7 @@ jobs:
python -VV
python -m site
python -m pip install --upgrade pip setuptools wheel
python -m pip install --upgrade "tox<4" "tox-gh-actions<3"
python -m pip install --upgrade "tox" "tox-gh-actions"

- name: "Run tox targets for ${{ matrix.python-version }}"
env:
Expand All @@ -95,14 +95,19 @@ jobs:
continue-on-error: true

integration-tests:
name: "Integration tests"
name: "Integration tests (${{ matrix.broker }})"
needs: lint
runs-on: ubuntu-latest

strategy:
matrix:
# TODO Add back rabbitmq
broker: [redis]

services:
rabbitmq:
image: rabbitmq
# Set health checks to wait until redis has started
# Set health checks to wait until rabbitmq has started
options: >-
--health-cmd "rabbitmq-diagnostics -q status"
--health-interval 10s
Expand All @@ -122,21 +127,19 @@ jobs:
- 6379:6379

steps:
- uses: actions/checkout@v2
- uses: actions/checkout@v7

- uses: "actions/setup-python@v2"
- uses: actions/setup-python@v7
with:
python-version: "3.x"
python-version: "3.10"

- name: "Install dependencies"
run: |
set -xe
python -VV
python -m site
python -m pip install --upgrade pip setuptools wheel
python -m pip install --upgrade "tox<4" "tox-gh-actions<3"
python -m pip install --upgrade "tox"

- name: "Run tox targets"
env:
TOX_SKIP_ENV: ".*unit.*|flake8"
run: "python -m tox"
run: "python -m tox -e 3.10-celery52-integration-${{ matrix.broker }}"
2 changes: 1 addition & 1 deletion .readthedocs.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ version: 2
formats: all

build:
os: ubuntu-20.04
os: ubuntu-24.04
tools:
python: "3.10"

Expand Down
2 changes: 2 additions & 0 deletions CHANGELOG.rst
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ Maintenance
-----------

* Drop support for Python 3.9 (`#106 <https://github.com/clokep/celery-batches/pull/106>`_)
* Update GitHub Actions and dev dependencies (flake8, mypy, isort, pyupgrade, black).
(`#110 <https://github.com/clokep/celery-batches/pull/110>`_)

2026-01-16
==========
Expand Down
44 changes: 17 additions & 27 deletions celery_batches/__init__.py
Original file line number Diff line number Diff line change
@@ -1,18 +1,8 @@
from collections.abc import Callable, Collection, Iterable
from itertools import count, filterfalse, tee
from queue import Empty, Queue
from time import monotonic
from typing import (
Any,
Callable,
Collection,
Dict,
Iterable,
NoReturn,
Optional,
Set,
Tuple,
TypeVar,
)
from typing import Any, NoReturn, TypeVar

from celery_batches.trace import apply_batches_task

Expand Down Expand Up @@ -67,7 +57,7 @@ def consume_queue(queue: "Queue[T]") -> Iterable[T]:

def partition(
predicate: Callable[[T], bool], iterable: Iterable[T]
) -> Tuple[Iterable[T], Iterable[T]]:
) -> tuple[Iterable[T], Iterable[T]]:
"Use a predicate to partition entries into false entries and true entries"
t1, t2 = tee(iterable)
return filterfalse(predicate, t1), filter(predicate, t2)
Expand All @@ -91,10 +81,10 @@ class SimpleRequest:
name = None

#: positional arguments
args: Tuple[Any, ...] = ()
args: tuple[Any, ...] = ()

#: keyword arguments
kwargs: Dict[Any, Any] = {}
kwargs: dict[Any, Any] = {}

#: message delivery information.
delivery_info = None
Expand All @@ -112,7 +102,7 @@ class SimpleRequest:
correlation_id = None

#: includes all of the original request headers
request_dict: Optional[Dict[str, Any]] = {}
request_dict: dict[str, Any] | None = {}

#: TODO
chord = None
Expand All @@ -121,14 +111,14 @@ def __init__(
self,
id: str,
name: str,
args: Tuple[Any, ...],
kwargs: Dict[Any, Any],
args: tuple[Any, ...],
kwargs: dict[Any, Any],
delivery_info: dict,
hostname: str,
ignore_result: bool,
reply_to: Optional[str],
correlation_id: Optional[str],
request_dict: Optional[Dict[str, Any]],
reply_to: str | None,
correlation_id: str | None,
request_dict: dict[str, Any] | None,
):
self.id = id
self.name = name
Expand Down Expand Up @@ -182,7 +172,7 @@ def __init__(self) -> None:
self._buffer: Queue[Request] = Queue()
self._pending: Queue[Request] = Queue()
self._count = count(1)
self._tref: Optional[Timer] = None
self._tref: Timer | None = None
self._pool: BasePool = None

def run(self, *args: Any, **kwargs: Any) -> NoReturn:
Expand Down Expand Up @@ -228,10 +218,10 @@ def Strategy(self, task: "Batches", app: Celery, consumer: Consumer) -> Callable

def task_message_handler(
message: Message,
body: Optional[Dict[str, Any]],
body: dict[str, Any] | None,
ack: promise,
reject: promise,
callbacks: Set,
callbacks: set,
**kw: Any,
) -> None:
if body is None and "args" not in message.payload:
Expand Down Expand Up @@ -277,8 +267,8 @@ def task_message_handler(

def apply(
self,
args: Optional[Tuple[Any, ...]] = None,
kwargs: Optional[dict] = None,
args: tuple[Any, ...] | None = None,
kwargs: dict | None = None,
*_args: Any,
**options: Any,
) -> Any:
Expand Down Expand Up @@ -371,7 +361,7 @@ def on_accepted(pid: int, time_accepted: float) -> None:
for req in acks_early:
req.acknowledge()

def on_return(result: Optional[Any]) -> None:
def on_return(result: Any | None) -> None:
for req in acks_late:
req.acknowledge()

Expand Down
4 changes: 2 additions & 2 deletions celery_batches/trace.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
Mimics some of the functionality found in celery.app.trace.trace_task.
"""

from typing import TYPE_CHECKING, Any, List, Tuple
from typing import TYPE_CHECKING, Any

from celery import signals, states
from celery._state import _task_stack
Expand All @@ -27,7 +27,7 @@


def apply_batches_task(
task: "Batches", args: Tuple[List["SimpleRequest"]], loglevel: int, logfile: None
task: "Batches", args: tuple[list["SimpleRequest"]], loglevel: int, logfile: None
) -> Any:
request_stack = task.request_stack
push_request = request_stack.push
Expand Down
10 changes: 5 additions & 5 deletions requirements/pkgutils.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
pyupgrade==2.31.1
flake8==4.0.1
isort==5.10.1
black==24.3.0
mypy==0.942
pyupgrade==3.21.2
flake8==7.3.0
isort==5.13.2
black==26.5.1
mypy==2.3.1
4 changes: 2 additions & 2 deletions t/integration/conftest.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import os
from typing import Any, Dict
from typing import Any

import pytest
from _pytest.fixtures import SubRequest
Expand All @@ -9,7 +9,7 @@


@pytest.fixture(scope="session", params=[1, 2])
def celery_config(request: SubRequest) -> Dict[str, Any]:
def celery_config(request: SubRequest) -> dict[str, Any]:
return {
"broker_url": TEST_BROKER,
"result_backend": TEST_BACKEND,
Expand Down
6 changes: 2 additions & 4 deletions t/integration/tasks.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,3 @@
from typing import List

from celery_batches import Batches, SimpleRequest

from celery import shared_task
Expand All @@ -9,7 +7,7 @@


@shared_task(base=Batches, flush_every=2, flush_interval=0.1)
def add(requests: List[SimpleRequest]) -> int:
def add(requests: list[SimpleRequest]) -> int:
"""
Add the first argument of each task.

Expand All @@ -29,7 +27,7 @@ def add(requests: List[SimpleRequest]) -> int:


@shared_task(base=Batches, flush_every=2, flush_interval=0.1)
def cumadd(requests: List[SimpleRequest]) -> None:
def cumadd(requests: list[SimpleRequest]) -> None:
"""
Calculate the cumulative sum of the first argument of each task.

Expand Down
13 changes: 7 additions & 6 deletions t/integration/test_batches.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
from collections.abc import Callable
from datetime import datetime, timedelta
from time import sleep
from typing import Any, Callable, List, Optional, Union
from typing import Any

from celery_batches import Batches, SimpleRequest

Expand All @@ -23,15 +24,15 @@ def __init__(
self,
signal: Signal,
expected_calls: int,
callback: Optional[Callable[..., None]] = None,
callback: Callable[..., None] | None = None,
):
self.signal = signal
signal.connect(self)
self.calls = 0
self.expected_calls = expected_calls
self.callback = callback

def __call__(self, sender: Union[Task, str, Consumer], **kwargs: Any) -> None:
def __call__(self, sender: Task | str | Consumer, **kwargs: Any) -> None:
if isinstance(sender, Task):
task_name = sender.name
elif isinstance(sender, Consumer):
Expand Down Expand Up @@ -221,7 +222,7 @@ def test_signals(celery_app: Celery, celery_worker: TestWorkController) -> None:
def test_current_task(celery_app: Celery, celery_worker: TestWorkController) -> None:
"""Ensure the current_task is properly set when running the task."""

def signal(sender: Union[Task, str], **kwargs: Any) -> None:
def signal(sender: Task | str, **kwargs: Any) -> None:
assert celery_app.current_task.name == "t.integration.tasks.add"

counter = SignalCounter(signals.task_prerun, 1, signal)
Expand Down Expand Up @@ -252,7 +253,7 @@ def acknowledge(self) -> None:
@celery_app.task(
base=Batches, flush_every=2, flush_interval=0.1, Request=AckRequest
)
def acks(requests: List[SimpleRequest]) -> None:
def acks(requests: list[SimpleRequest]) -> None:
# The tasks are acked before running.
assert acked == [result_1.id, result_2.id]

Expand Down Expand Up @@ -291,7 +292,7 @@ def acknowledge(self) -> None:
flush_interval=0.1,
Request=AckRequest,
)
def acks(requests: List[SimpleRequest]) -> None:
def acks(requests: list[SimpleRequest]) -> None:
# When the tasks are running, nothing is acked.
assert acked == []

Expand Down