Add a global docker cleanup service

Author: azvoleff

docker-compose.prod.yml

@@ -189,6 +189,46 @@ services:
         max_attempts: 3
         window: 120s
 
+  # Global Docker cleanup service - runs on ALL nodes (managers + workers)
+  # This ensures images, containers, and build cache are cleaned on every node,
+  # not just managers where the Celery cleanup tasks run.
+  # Security note: Uses minimal docker:24-cli image with only prune commands.
+  docker-cleanup:
+    image: docker:24-cli
+    user: root
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    # Run cleanup every 6 hours: prune containers, dangling images, and build cache
+    # Uses alpine's built-in sleep; runs indefinitely
+    command: |
+      sh -c 'while true; do
+        echo "[$(date)] Starting Docker cleanup on node..."
+        echo "Pruning stopped containers..."
+        docker container prune -f
+        echo "Pruning dangling images..."
+        docker image prune -f
+        echo "Pruning build cache (older than 168h)..."
+        docker builder prune -f --filter "until=168h" || true
+        echo "Pruning unused volumes..."
+        docker volume prune -f || true
+        echo "[$(date)] Cleanup complete. Sleeping 6 hours..."
+        sleep 21600
+      done'
+    deploy:
+      mode: global
+      resources:
+        reservations:
+          cpus: "0.1"
+          memory: 50M
+        limits:
+          cpus: "0.5"
+          memory: 100M
+      restart_policy:
+        condition: on-failure
+        delay: 60s
+        max_attempts: 3
+        window: 300s
+
   redis:
     image: redis:7-alpine
     volumes:

docker-compose.staging.yml

@@ -214,6 +214,45 @@ services:
         max_attempts: 3
         window: 120s
 
+  # Global Docker cleanup service - runs on ALL nodes (managers + workers)
+  # This ensures images, containers, and build cache are cleaned on every node,
+  # not just managers where the Celery cleanup tasks run.
+  # Security note: Uses minimal docker:24-cli image with only prune commands.
+  docker-cleanup:
+    image: docker:24-cli
+    user: root
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+    # Run cleanup every 6 hours: prune containers, dangling images, and build cache
+    command: |
+      sh -c 'while true; do
+        echo "[$(date)] Starting Docker cleanup on node..."
+        echo "Pruning stopped containers..."
+        docker container prune -f
+        echo "Pruning dangling images..."
+        docker image prune -f
+        echo "Pruning build cache (older than 168h)..."
+        docker builder prune -f --filter "until=168h" || true
+        echo "Pruning unused volumes..."
+        docker volume prune -f || true
+        echo "[$(date)] Cleanup complete. Sleeping 6 hours..."
+        sleep 21600
+      done'
+    deploy:
+      mode: global  # Run on EVERY node in the swarm
+      resources:
+        reservations:
+          cpus: "0.1"
+          memory: 50M
+        limits:
+          cpus: "0.5"
+          memory: 100M
+      restart_policy:
+        condition: on-failure
+        delay: 60s
+        max_attempts: 3
+        window: 300s
+
   postgres:
     image: postgis/postgis:16-3.4
     environment:

gefapi/celery.py

@@ -55,10 +55,6 @@ def make_celery(app):
         "gefapi.tasks.docker_completed_monitoring.monitor_completed_docker_services": {
             "queue": "build"
         },
-        # Docker resource cleanup task - run on build queue with Docker access
-        "gefapi.tasks.docker_resource_cleanup.cleanup_docker_build_cache": {
-            "queue": "build"
-        },
         # Stats cache refresh tasks - run on default queue
         "gefapi.tasks.stats_cache_refresh.refresh_dashboard_stats_cache": {
             "queue": "default"
@@ -130,11 +126,6 @@ def make_celery(app):
             "schedule": 180.0,  # Every 3 minutes - check for completed services
             "options": {"queue": "build"},  # Run on build queue with Docker access
         },
-        "cleanup-docker-build-cache": {
-            "task": "gefapi.tasks.docker_resource_cleanup.cleanup_docker_build_cache",
-            "schedule": 604800.0,  # Every week (7 days = 604800 seconds)
-            "options": {"queue": "build"},  # Run on build queue with Docker access
-        },
         # Stats cache refresh tasks for performance optimization
         "refresh-dashboard-stats-cache": {
             "task": "gefapi.tasks.stats_cache_refresh.refresh_dashboard_stats_cache",