Finalise fixes to photo uploader workflow (separate dialogs, review map, edge cases & resiliency)

compose.test.yaml

@@ -1,15 +1,14 @@
-# compose.test.yaml
 name: dtm-test
 
+networks:
+  dtm-network:
+    name: dtm-network
+
 volumes:
   db-data:
   minio-data:
   frontend-html:
 
-networks:
-  dtm-network:
-    driver: bridge
-
 services:
   backend:
     extends:
@@ -49,17 +48,11 @@ services:
       service: createbuckets
 
   migrations:
-    image: ghcr.io/hotosm/drone-tm/backend:debug
-    volumes:
-      - ./src/backend:/project/src/backend
-    env_file:
-      - .env
+    extends:
+      file: compose.yaml
+      service: migrations
     environment:
       MONITORING: ""
-    networks:
-      - dtm-network
-    command: ["alembic", "upgrade", "head"]
-    restart: "no"
 
   arq-worker:
     extends:

compose.yaml

@@ -177,7 +177,7 @@ services:
   # This container does the actual imagery processing (not persistent, scalable)
   nodeodm:
     image: docker.io/opendronemap/nodeodm:3.5.5
-    command: ["--port", "9900"]
+    command: ["--port", "9900", "--log_level", "debug"]
     env_file: .env
     ports:
       - 9900:9900

src/backend/app/arq/tasks.py

@@ -20,7 +20,12 @@
 from app.images.image_schemas import ProjectImageCreate, ProjectImageOut
 from app.images.flight_tail_removal import mark_and_remove_flight_tail_imagery
 from app.models.enums import HTTPStatus, ImageStatus
-from app.projects.project_logic import process_all_drone_images, process_drone_images
+from app.projects import project_schemas
+from app.projects.project_logic import (
+    process_all_drone_images,
+    process_drone_images,
+    process_task_metrics,
+)
 from app.s3 import async_get_obj_from_bucket, s3_client
 from app.images.image_classification import ImageClassifier
 from app.jaxa.upload_dem import download_and_upload_dem
@@ -620,78 +625,71 @@ async def delete_batch_images(
 
     try:
         async with db_pool.connection() as conn:
-            # Get all S3 keys for images and thumbnails in this batch
-            query = """
-                SELECT s3_key, thumbnail_url
-                FROM project_images
-                WHERE batch_id = %(batch_id)s
-                AND project_id = %(project_id)s
-            """
+            result = await ImageClassifier.delete_batch(
+                conn, UUID(batch_id), UUID(project_id)
+            )
 
-            async with conn.cursor() as cur:
-                await cur.execute(
-                    query,
-                    {"batch_id": batch_id, "project_id": project_id},
-                )
-                rows = await cur.fetchall()
-
-            # Collect all S3 keys to delete
-            s3_keys_to_delete = []
-            for row in rows:
-                s3_key, thumbnail_url = row
-                if s3_key:
-                    s3_keys_to_delete.append(s3_key)
-                if thumbnail_url:
-                    s3_keys_to_delete.append(thumbnail_url)
-
-            image_count = len(rows)
             log.info(
-                f"Found {image_count} images and {len(s3_keys_to_delete)} S3 objects to delete"
+                f"Batch deletion complete: {result['deleted_count']} images, "
+                f"{result['deleted_s3_count']} S3 objects deleted"
             )
 
-            # Delete from S3
-            deleted_s3_count = 0
-            if s3_keys_to_delete:
-                client = s3_client()
-                for key in s3_keys_to_delete:
-                    try:
-                        key = key.lstrip("/")
-                        client.remove_object(settings.S3_BUCKET_NAME, key)
-                        deleted_s3_count += 1
-                    except Exception as e:
-                        log.warning(f"Failed to delete S3 object {key}: {e}")
-
-            log.info(f"Deleted {deleted_s3_count} objects from S3")
-
-            # Delete from database
-            delete_query = """
-                DELETE FROM project_images
-                WHERE batch_id = %(batch_id)s
-                AND project_id = %(project_id)s
-            """
+            return {
+                "message": result["message"],
+                "batch_id": batch_id,
+                "deleted_images": result["deleted_count"],
+                "deleted_s3_objects": result["deleted_s3_count"],
+            }
 
-            async with conn.cursor() as cur:
+    except Exception as e:
+        log.error(f"Failed to delete batch (Job: {job_id}): {str(e)}")
+        raise
+
+
+async def process_project_task_metrics(
+    ctx: Dict[Any, Any], project_id: str
+) -> Dict[str, Any]:
+    """Process project task metrics in the ARQ worker."""
+    job_id = ctx.get("job_id", "unknown")
+    log.info(
+        f"Starting process_project_task_metrics (Job ID: {job_id}): project={project_id}"
+    )
+
+    db_pool = ctx.get("db_pool")
+    if not db_pool:
+        raise RuntimeError("Database pool not initialized in ARQ context")
+
+    try:
+        async with db_pool.connection() as db:
+            project = await project_schemas.DbProject.one(db, UUID(project_id))
+
+            async with db.cursor() as cur:
                 await cur.execute(
-                    delete_query,
-                    {"batch_id": batch_id, "project_id": project_id},
+                    """
+                    SELECT id, project_id, ST_AsBinary(outline), project_task_index
+                    FROM tasks
+                    WHERE project_id = %s
+                    ORDER BY project_task_index
+                    """,
+                    (project.id,),
                 )
+                tasks_data = await cur.fetchall()
 
-            await conn.commit()
+            await process_task_metrics(db, tasks_data, project)
 
             log.info(
-                f"Batch deletion complete: {image_count} images, "
-                f"{deleted_s3_count} S3 objects deleted"
+                f"Completed process_project_task_metrics (Job ID: {job_id}): "
+                f"project={project_id}, tasks={len(tasks_data)}"
             )
 
             return {
-                "message": "Batch deleted successfully",
-                "batch_id": batch_id,
-                "deleted_images": image_count,
-                "deleted_s3_objects": deleted_s3_count,
+                "message": "Task metrics processed",
+                "project_id": project_id,
+                "task_count": len(tasks_data),
             }
 
     except Exception as e:
-        log.error(f"Failed to delete batch (Job: {job_id}): {str(e)}")
+        log.error(f"Failed process_project_task_metrics (Job ID: {job_id}): {str(e)}")
         raise
 
 
@@ -708,6 +706,7 @@ class WorkerSettings:
         classify_image_batch,
         process_batch_images,
         delete_batch_images,
+        process_project_task_metrics,
         download_and_upload_dem,
     ]