Remove starting the supervisor locally

Starting the collector and processor together locally is a foot-gun -
it's going to go off and start doing things on production. There's
not much point in doing so, so remove the option, but add:

  make supervisor-collect: collect once
  make supervisor-clear-queue: clear the queue

So that the collector can be tested locally (testing the processor
is best done with process-issue and process-errratum)

Author: owtaylor

Makefile

@@ -173,18 +173,6 @@ trigger-pipeline:
 
 # Testing and Release Supervisor
 
-.PHONY: supervisor-start
-supervisor-start:
-	DRY_RUN=$(DRY_RUN) $(COMPOSE_SUPERVISOR) up
-
-.PHONY: supervisor-start-detached
-supervisor-start-detached:
-	DRY_RUN=$(DRY_RUN) $(COMPOSE_SUPERVISOR) up -d
-
-.PHONY: supervisor-stop
-supervisor-stop:
-	$(COMPOSE_SUPERVISOR) down
-
 DEBUG_LOWER := $(shell echo $(DEBUG) | tr '[:upper:]' '[:lower:]')
 ifeq ($(DEBUG_LOWER),true)
 DEBUG_FLAG := --debug
@@ -199,6 +187,16 @@ else
 DRY_RUN_FLAG :=
 endif
 
+.PHONY: supervisor-clear-queue
+supervisor-clear-queue:
+	$(COMPOSE_SUPERVISOR) run --rm \
+		supervisor python -m supervisor.main $(DEBUG_FLAG) clear-queue
+
+.PHONY: supervisor-collect
+supervisor-collect:
+	$(COMPOSE_SUPERVISOR) run --rm \
+		supervisor python -m supervisor.main $(DEBUG_FLAG) collect --no-repeat
+
 .PHONY: process-issue
 process-issue:
 	$(COMPOSE_SUPERVISOR) run --rm \

README-supervisor.md

@@ -58,10 +58,14 @@ DRY_RUN=true # Don't actually make any changes to issues/errata, just show what
 DEBUG=true   # Use more detailed logging (shows exactly what would happen for dry run items)
 ```
 
-## Starting the collector and processor
+## Testing the collector
+
+You can test the collection process with:
 
 ```
-make supervisor-start
+make supervisor-clear-queue
+make supervisor-collect
 ```
 
-This runs the services in the foreground, showing logs for monitoring and debugging. If you prefer to run the services in the background, use `make supervisor-start-detached` instead.
+That will clear any existing queue items, collect items from scratch,
+and log what gets added to the queue.

compose.yaml

@@ -151,16 +151,6 @@ services:
     command: ["python", "agents/rebase_agent.py"]
     profiles: ["agents"]
 
-  supervisor-collector:
-    <<: *supervisor
-    command: ["python", "-m", "supervisor.main", "collect"]
-    profiles: ["supervisor"]
-
-  supervisor-processor:
-    <<: *supervisor
-    command: ["python", "-m", "supervisor.main", "process"]
-    profiles: ["supervisor"]
-
   # This is here so we can use it for '$(COMPOSE) run'
   supervisor:
     <<: *supervisor

supervisor/main.py

@@ -123,6 +123,19 @@ def collect(
     asyncio.run(do_collect(repeat, repeat_delay))
 
 
+async def do_clear_queue():
+    async with work_queue(os.environ["REDIS_URL"]) as queue:
+        await queue.remove_all_work_items()
+        logger.info("Cleared the work item queue")
+
+
+@app.command()
+def clear_queue():
+    check_env(redis=True)
+
+    asyncio.run(do_clear_queue())
+
+
 @with_http_sessions()
 async def process_once(queue: WorkQueue):
     work_item = await queue.wait_first_ready_work_item()

supervisor/work_queue.py

@@ -98,6 +98,9 @@ async def remove_work_items(self, work_items: Iterable[WorkItem]) -> None:
 
         await self.client.zrem("supervisor_work_queue", *to_remove)
 
+    async def remove_all_work_items(self) -> None:
+        await self.client.delete("supervisor_work_queue")
+
     async def get_all_work_items(self) -> list[WorkItem]:
         work_items = await self.client.zrange("supervisor_work_queue", 0, -1)
         return [