Serialize cluster image builds and split CLI tests into separate CI job (#338)

Two changes to improve CI reliability:

**Serialize cluster image builds with file lock**

The `AlreadyExists` fix in #337 handled one symptom of parallel xdist
workers racing to build the same cluster image, but there's a second
failure mode showing up in CI:


https://github.com/chrisguidry/docket/actions/runs/22025132964/job/63640478732

When concurrent builds target the same tag, the Docker SDK's `build()`
completes successfully in the daemon, then tries to inspect the
resulting image by its short ID. If another worker's build re-tagged the
image in the meantime, the first image ID gets orphaned and the inspect
404s. This knocked out 485 of 686 tests in the cluster job.

Rather than catching yet another exception type, this serializes the
builds with `fcntl.flock` so only one worker builds at a time. The
others wait and find it already built. Eliminates both the
`AlreadyExists` and `ImageNotFound` races structurally.

**Split CLI tests into separate CI job**

Cluster CI jobs consistently run right at the 4-minute timeout, and when
any test runs slightly slow the whole job gets cancelled. This has been
showing up in roughly a third of recent CI runs:


https://github.com/chrisguidry/docket/actions/runs/22025359927/job/63641245074

The 91 CLI tests are subprocess-based and don't exercise
backend-specific behavior — they spawn `python -m docket ...` processes
and check output. Running them against every Python x Backend
combination (30 matrix entries) is wasted effort.

This moves CLI tests to their own job that varies by Python version but
uses a single Redis backend (8.0). The main test matrix now passes
`--ignore=tests/cli` so cluster/valkey/memory jobs only run the tests
that actually care about the backend. Local `pytest` runs are
unaffected.

---------

Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>

Author: chrisguidry

.coveragerc-cli

@@ -0,0 +1,9 @@
+[run]
+branch = true
+parallel = true
+
+[report]
+include =
+    src/docket/cli/*
+    tests/cli/*
+exclude_also = ["\\.\\.\\.$"]

.coveragerc-core

@@ -0,0 +1,16 @@
+[run]
+branch = true
+parallel = true
+omit =
+    src/docket/__main__.py
+    src/docket/_uuid7.py
+    src/docket/_prometheus_exporter.py
+    src/docket/_result_store.py
+    src/docket/cli/*.py
+    **/conftest.py
+    tests/_key_leak_checker.py
+    tests/_container.py
+    tests/cli/*.py
+
+[report]
+exclude_also = ["\\.\\.\\.$"]

.coveragerc-memory

@@ -1,3 +1,12 @@
+ignore:
+  - "src/docket/__main__.py"
+  - "src/docket/_uuid7.py"
+  - "src/docket/_prometheus_exporter.py"
+  - "src/docket/_result_store.py"
+  - "**/conftest.py"
+  - "tests/_key_leak_checker.py"
+  - "tests/_container.py"
+
 coverage:
   status:
     project:

.github/codecov.yml

@@ -11,7 +11,7 @@ permissions:
 
 jobs:
   test:
-    name: Test Python ${{ matrix.python-version }}, ${{ matrix.backend.name }}
+    name: Core Tests, Python ${{ matrix.python-version }}, ${{ matrix.backend.name }}
     runs-on: ubuntu-latest
     timeout-minutes: 4
     strategy:
@@ -54,15 +54,14 @@ jobs:
           - python-version: "3.14"
             cov-threshold: 100
             pytest-args: ""
-          # Memory backend: CLI tests are skipped via pytest skip markers because
-          # CLI rejects memory:// URLs. Use separate coverage config to exclude CLI.
-          # Coverage threshold is lower due to pytest-xdist coverage data combination issues.
+          # Memory backend: coverage threshold is lower due to pytest-xdist
+          # coverage data combination issues.
           - backend:
               name: "Memory (in-memory backend)"
               redis-version: "memory"
               redis-py-version: ">=5"
             cov-threshold: 95
-            pytest-args: "--cov-config=.coveragerc-memory"
+            pytest-args: ""
           # Debug hanging tests in 3.12 cluster mode
           - python-version: "3.12"
             backend:
@@ -129,14 +128,54 @@ jobs:
       - name: Run tests
         env:
           REDIS_VERSION: ${{ matrix.backend.redis-version }}
-        run: uv run pytest --cov-branch --cov-fail-under=${{ matrix.cov-threshold }} --cov-report=xml --cov-report=term-missing:skip-covered ${{ matrix.pytest-args }}
+        run: uv run pytest --ignore=tests/cli --cov-branch --cov-config=.coveragerc-core --cov-fail-under=${{ matrix.cov-threshold }} --cov-report=xml --cov-report=term-missing:skip-covered ${{ matrix.pytest-args }}
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
           flags: python-${{ matrix.python-version }}
 
+  test-cli:
+    name: CLI Tests, Python ${{ matrix.python-version }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 4
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - uses: actions/checkout@v6
+
+      - name: Install uv and set Python version
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: ${{ matrix.python-version }}
+          enable-cache: true
+          cache-dependency-glob: "pyproject.toml"
+
+      - name: Install dependencies
+        run: uv sync --upgrade-package 'redis>=5'
+
+      - name: Run CLI tests
+        env:
+          REDIS_VERSION: "8.0"
+        run: >
+          uv run pytest tests/cli/
+          -o 'addopts=--import-mode=importlib'
+          --numprocesses=logical --maxprocesses=4
+          --cov=src/docket/cli --cov=tests/cli
+          --cov-config=.coveragerc-cli
+          --cov-branch --cov-fail-under=100
+          --cov-report=xml --cov-report=term-missing:skip-covered
+          --timeout=30
+
+      - name: Upload coverage reports to Codecov
+        uses: codecov/codecov-action@v5
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: cli-python-${{ matrix.python-version }}
+
   prek:
     name: Prek checks
     runs-on: ubuntu-latest

.github/workflows/ci.yml

@@ -13,7 +13,7 @@ path = "src/docket/worker.py"
 max_lines = 1126
 
 [[rules]]
-path = "src/docket/cli.py"
+path = "src/docket/cli/__init__.py"
 max_lines = 945
 
 [[rules]]

loq.toml

@@ -115,6 +115,7 @@ addopts = [
     "--cov=tests",
     "--cov-report=term-missing:skip-covered",
     "--cov-branch",
+    "--cov-fail-under=100",
     "--timeout=30",
     "--import-mode=importlib",
 ]

pyproject.toml

@@ -19,8 +19,8 @@
 )
 from rich.table import Table
 
-from . import __version__, tasks
-from ._cli_support import (
+from docket import __version__, tasks
+from docket.cli._support import (
     LogLevel,
     LogFormat,
     default_worker_name,
@@ -36,10 +36,10 @@
     set_logging_level,
     validate_url,
 )
-from .docket import Docket, DocketSnapshot, WorkerInfo
-from .execution import ExecutionState
-from .strikelist import Operator
-from .worker import Worker
+from docket.docket import Docket, DocketSnapshot, WorkerInfo
+from docket.execution import ExecutionState
+from docket.strikelist import Operator
+from docket.worker import Worker
 
 
 app: typer.Typer = typer.Typer(

src/docket/cli.py src/docket/cli/__init__.pysrc/docket/cli.py renamed to src/docket/cli/__init__.py

@@ -15,7 +15,7 @@
 from rich.console import Console
 from rich.table import Table
 
-from .docket import DocketSnapshot, WorkerInfo
+from docket.docket import DocketSnapshot, WorkerInfo
 
 
 async def iterate_with_timeout(

src/docket/_cli_support.py src/docket/cli/_support.pysrc/docket/_cli_support.py renamed to src/docket/cli/_support.py

@@ -4,8 +4,10 @@
 including single-node Redis, Redis Cluster, and Valkey variants.
 """
 
+import fcntl
 import os
 import socket
+import tempfile
 import time
 from contextlib import contextmanager
 from datetime import datetime, timedelta, timezone
@@ -167,7 +169,13 @@ def wrapper(*args: P.args, **kwargs: P.kwargs) -> T:
 
 
 def build_cluster_image(client: DockerClient, base_image: str) -> str:
-    """Build cluster image from base image, return image tag."""
+    """Build cluster image from base image, return image tag.
+
+    Uses a file lock to serialize builds across parallel xdist workers.
+    Without the lock, concurrent builds for the same tag race: one build
+    re-tags the image and the Docker SDK's post-build inspect on the
+    loser's image ID gets a 404.
+    """
     tag = f"docket-cluster:{base_image.replace('/', '-').replace(':', '-')}"
 
     try:
@@ -176,18 +184,23 @@ def build_cluster_image(client: DockerClient, base_image: str) -> str:
     except docker.errors.ImageNotFound:
         pass
 
-    cluster_dir = Path(__file__).parent / "cluster"
-    try:
+    lock_path = Path(tempfile.gettempdir()) / f"docket-{tag.replace(':', '-')}.lock"
+    with open(lock_path, "w") as lock_file:
+        fcntl.flock(lock_file, fcntl.LOCK_EX)
+
+        # Re-check after acquiring lock; another worker may have built it
+        try:
+            client.images.get(tag)
+            return tag
+        except docker.errors.ImageNotFound:
+            pass
+
+        cluster_dir = Path(__file__).parent / "cluster"
         client.images.build(
             path=str(cluster_dir),
             tag=tag,
             buildargs={"BASE_IMAGE": base_image},
         )
-    except docker.errors.BuildError as e:
-        if "AlreadyExists" in str(e):
-            pass
-        else:
-            raise
     return tag