feat(backend): Move Scheduler (#9904)

<!-- Clearly explain the need for these changes: -->
We want the scheduler shouldn't scale with the rest API lol

### Changes 🏗️
pulls out the scheduler into its own service
<!-- Concisely describe all of the changes made in this pull request:
-->

### Checklist 📋

#### For code changes:
- [x] I have clearly listed my changes in the PR description
- [x] I have made a test plan
- [x] I have tested my changes according to the test plan:
  <!-- Put your test plan here: -->
  - [x] test it

---------

Co-authored-by: Zamil Majdy <[email protected]>

Author: ntindle

autogpt_platform/backend/backend/rest.py

@@ -1,5 +1,5 @@
 from backend.app import run_processes
-from backend.executor import DatabaseManager, Scheduler
+from backend.executor import DatabaseManager
 from backend.notifications.notifications import NotificationManager
 from backend.server.rest_api import AgentServer
 
@@ -11,7 +11,6 @@ def main():
     run_processes(
         NotificationManager(),
         DatabaseManager(),
-        Scheduler(),
         AgentServer(),
     )
 

autogpt_platform/backend/backend/scheduler.py

@@ -0,0 +1,13 @@
+from backend.app import run_processes
+from backend.executor.scheduler import Scheduler
+
+
+def main():
+    """
+    Run all the processes required for the AutoGPT-server Scheduling System.
+    """
+    run_processes(Scheduler())
+
+
+if __name__ == "__main__":
+    main()

autogpt_platform/backend/pyproject.toml

@@ -88,6 +88,7 @@ build-backend = "poetry.core.masonry.api"
 app = "backend.app:main"
 rest = "backend.rest:main"
 ws = "backend.ws:main"
+scheduler = "backend.scheduler:main"
 executor = "backend.exec:main"
 cli = "backend.cli:main"
 format = "linter:format"

autogpt_platform/docker-compose.platform.yml

@@ -73,6 +73,8 @@ services:
         condition: service_completed_successfully
       rabbitmq:
         condition: service_healthy
+      # scheduler_server:
+      #   condition: service_healthy
     environment:
       - SUPABASE_URL=http://kong:8000
       - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long
@@ -88,7 +90,7 @@ services:
       - REDIS_PASSWORD=password
       - ENABLE_AUTH=true
       - PYRO_HOST=0.0.0.0
-      - EXECUTIONSCHEDULER_HOST=rest_server
+      - SCHEDULER_HOST=scheduler_server
       - EXECUTIONMANAGER_HOST=executor
       - NOTIFICATIONMANAGER_HOST=rest_server
       - FRONTEND_BASE_URL=http://localhost:3000
@@ -98,7 +100,6 @@ services:
     ports:
       - "8006:8006"
       - "8007:8007"
-      - "8003:8003" # execution scheduler
     networks:
       - app-network
 
@@ -187,6 +188,61 @@ services:
     networks:
       - app-network
 
+  scheduler_server:
+    build:
+      context: ../
+      dockerfile: autogpt_platform/backend/Dockerfile
+      target: server
+    command: ["python", "-m", "backend.scheduler"]
+    develop:
+      watch:
+        - path: ./
+          target: autogpt_platform/backend/
+          action: rebuild
+    depends_on:
+      db:
+        condition: service_healthy
+      redis:
+        condition: service_healthy
+      rabbitmq:
+        condition: service_healthy
+      migrate:
+        condition: service_completed_successfully
+    # healthcheck:
+    #   test:
+    #     [
+    #       "CMD",
+    #       "curl",
+    #       "-f",
+    #       "-X",
+    #       "POST",
+    #       "http://localhost:8003/health_check",
+    #     ]
+    #   interval: 10s
+    #   timeout: 10s
+    #   retries: 5
+    environment:
+      - DATABASEMANAGER_HOST=rest_server
+      - NOTIFICATIONMANAGER_HOST=rest_server
+      - SUPABASE_JWT_SECRET=your-super-secret-jwt-token-with-at-least-32-characters-long
+      - DATABASE_URL=postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres?connect_timeout=60&schema=platform
+      - DIRECT_URL=postgresql://postgres:your-super-secret-and-long-postgres-password@db:5432/postgres?connect_timeout=60&schema=platform
+      - REDIS_HOST=redis
+      - REDIS_PORT=6379
+      - REDIS_PASSWORD=password
+      - RABBITMQ_HOST=rabbitmq
+      - RABBITMQ_PORT=5672
+      - RABBITMQ_DEFAULT_USER=rabbitmq_user_default
+      - RABBITMQ_DEFAULT_PASS=k0VMxyIJF9S35f3x2uaw5IWAl6Y536O7
+      - ENABLE_AUTH=true
+      - PYRO_HOST=0.0.0.0
+      - BACKEND_CORS_ALLOW_ORIGINS=["http://localhost:3000"]
+
+    ports:
+      - "8003:8003"
+    networks:
+      - app-network
+
 #  frontend:
 #    build:
 #      context: ../

autogpt_platform/docker-compose.yml

@@ -57,11 +57,17 @@ services:
       file: ./docker-compose.platform.yml
       service: websocket_server
 
-#  frontend:
-#    <<: *agpt-services
-#    extends:
-#      file: ./docker-compose.platform.yml
-#      service: frontend
+  scheduler_server:
+    <<: *agpt-services
+    extends:
+      file: ./docker-compose.platform.yml
+      service: scheduler_server
+
+  #  frontend:
+  #    <<: *agpt-services
+  #    extends:
+  #      file: ./docker-compose.platform.yml
+  #      service: frontend
 
   # Supabase services
   studio:

docs/content/platform/getting-started.md

@@ -382,7 +382,7 @@ Currently, there are only 3 active services:
 
 - AgentServer (the API, defined in `server.py`)
 - ExecutionManager (the executor, defined in `manager.py`)
-- ExecutionScheduler (the scheduler, defined in `scheduler.py`)
+- Scheduler (the scheduler, defined in `scheduler.py`)
 
 The services run in independent Python processes and communicate through an IPC.
 A communication layer (`service.py`) is created to decouple the communication library from the implementation.