Halve Ray cluster min_workers, boost Iris min_slices and controller VM

rjpower · rjpower · commit e31b2700e865 · 2026-03-26T09:41:06.000-07:00
Shift capacity from Ray to Iris to accelerate migration:

Ray min_workers halved across all clusters (training + vllm + staging).
Freed capacity given to Iris as min_slices:
- tpu_v5p_8: 0 → 8 (absorbed from us-central1 + us-east5-a)
- tpu_v4_8: 0 → 2 (absorbed from us-central2)
- tpu_v5e_4: 1 → 3 (absorbed from eu-west4)
- tpu_v5e_128: 0 → 1 (absorbed from eu-west4)
- tpu_v6e_128: 0 → 1 (absorbed from eu-west4-a)

Iris controller VM upgraded from e2-standard-4 (16GB) to e2-highmem-4 (32GB)
in both prod and dev configs to address memory pressure.
diff --git a/infra/marin-eu-west4-a.yaml b/infra/marin-eu-west4-a.yaml
@@ -194,7 +194,7 @@ available_node_types:
 
   tpu_slice_v6e_128:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v6e-128
       runtimeVersion: v2-alpha-tpuv6e
diff --git a/infra/marin-eu-west4-vllm.yaml b/infra/marin-eu-west4-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v5litepod-4
       runtimeVersion: v2-alpha-tpuv5-lite
diff --git a/infra/marin-eu-west4.yaml b/infra/marin-eu-west4.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 4
+    min_workers: 2
     node_config:
       acceleratorType: v5litepod-4
       runtimeVersion: v2-alpha-tpuv5-lite
@@ -194,7 +194,7 @@ available_node_types:
 
   tpu_slice_v5e_128:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5litepod-128
       runtimeVersion: v2-alpha-tpuv5-lite
diff --git a/infra/marin-us-central1-vllm.yaml b/infra/marin-us-central1-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -141,7 +141,7 @@ available_node_types:
 
   tpu_slice_v5p_8:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
diff --git a/infra/marin-us-central1.yaml b/infra/marin-us-central1.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -146,7 +146,7 @@ available_node_types:
 
   tpu_slice_v5p_8:
     max_workers: 1024
-    min_workers: 12
+    min_workers: 6
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -158,7 +158,7 @@ available_node_types:
 
   tpu_slice_v5p_16:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-16
       runtimeVersion: v2-alpha-tpuv5
@@ -170,7 +170,7 @@ available_node_types:
 
   tpu_slice_v5p_32:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-32
       runtimeVersion: v2-alpha-tpuv5
@@ -182,7 +182,7 @@ available_node_types:
 
   tpu_slice_v5p_64:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-64
       runtimeVersion: v2-alpha-tpuv5
diff --git a/infra/marin-us-central2-staging.yaml b/infra/marin-us-central2-staging.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 4
+    min_workers: 2
     node_config:
       acceleratorType: v4-8
       runtimeVersion: tpu-ubuntu2204-base
diff --git a/infra/marin-us-central2-vllm.yaml b/infra/marin-us-central2-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v4-8
       runtimeVersion: tpu-ubuntu2204-base
diff --git a/infra/marin-us-central2.yaml b/infra/marin-us-central2.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 4
+    min_workers: 2
     node_config:
       acceleratorType: v4-8
       runtimeVersion: tpu-ubuntu2204-base
diff --git a/infra/marin-us-east1-d-vllm.yaml b/infra/marin-us-east1-d-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v6e-8
       runtimeVersion: v2-alpha-tpuv6e
diff --git a/infra/marin-us-east5-a-vllm.yaml b/infra/marin-us-east5-a-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -141,7 +141,7 @@ available_node_types:
 
   tpu_slice_v5p_8:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
diff --git a/infra/marin-us-east5-a.yaml b/infra/marin-us-east5-a.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 8
+    min_workers: 4
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -146,7 +146,7 @@ available_node_types:
 
   tpu_slice_v5p_8:
     max_workers: 1024
-    min_workers: 8
+    min_workers: 4
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
diff --git a/infra/marin-us-east5-b-vllm.yaml b/infra/marin-us-east5-b-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v6e-8
       runtimeVersion: v2-alpha-tpuv6e
diff --git a/lib/iris/examples/marin-dev.yaml b/lib/iris/examples/marin-dev.yaml
@@ -29,7 +29,7 @@ controller:
   image: ghcr.io/marin-community/iris-controller:latest
   gcp:
     zone: us-central1-a
-    machine_type: e2-standard-4
+    machine_type: e2-highmem-4
     boot_disk_size_gb: 100
     port: 10000
 
diff --git a/lib/iris/examples/marin.yaml b/lib/iris/examples/marin.yaml
@@ -28,7 +28,7 @@ controller:
   image: ghcr.io/marin-community/iris-controller:latest
   gcp:
     zone: us-central1-a
-    machine_type: e2-standard-4
+    machine_type: e2-highmem-4
     boot_disk_size_gb: 100
     port: 10000
 
@@ -57,7 +57,7 @@ scale_groups:
     num_vms: 1
     priority: 10
     resources: { cpu: 112, ram: 192GB, disk: 100GB, device_type: tpu, device_variant: v5litepod-4, device_count: 4, preemptible: true }
-    min_slices: 1
+    min_slices: 3
     max_slices: 1024
     slice_template:
       gcp:
@@ -107,7 +107,7 @@ scale_groups:
     num_vms: 32
     priority: 60
     resources: { cpu: 112, ram: 192GB, disk: 100GB, device_type: tpu, device_variant: v5litepod-128, device_count: 4, preemptible: true }
-    min_slices: 0
+    min_slices: 1
     max_slices: 32
     slice_template:
       gcp:
@@ -182,7 +182,7 @@ scale_groups:
     num_vms: 32
     priority: 60
     resources: { cpu: 180, ram: 720GB, disk: 100GB, device_type: tpu, device_variant: v6e-128, device_count: 4, preemptible: true }
-    min_slices: 0
+    min_slices: 1
     max_slices: 32
     slice_template:
       gcp:
@@ -208,7 +208,7 @@ scale_groups:
     num_vms: 1
     priority: 20
     resources: { cpu: 208, ram: 448GB, disk: 100GB, device_type: tpu, device_variant: v5p-8, device_count: 4, preemptible: true }
-    min_slices: 0
+    min_slices: 8
     max_slices: 2048
     slice_template:
       gcp:
@@ -303,7 +303,7 @@ scale_groups:
     num_vms: 1
     priority: 20
     resources: { cpu: 240, ram: 400GB, disk: 100GB, device_type: tpu, device_variant: v4-8, device_count: 4, preemptible: true }
-    min_slices: 0
+    min_slices: 2
     max_slices: 2048
     slice_template:
       gcp:
