Halve Ray cluster minimums, boost Iris capacity (#4175)

rjpower · Helw150 · commit bfa22c6bfc98 · 2026-04-08T15:07:47.000-07:00
- Halved `min_workers` across all Ray clusters (12 files: training,
vllm, staging) to free TPU capacity
- Transferred freed capacity to Iris `min_slices` in `marin.yaml`: v5p-8
+8, v4-8 +2, v5e-4 +2, v5e-128 +1, v6e-128 +1
- Upgraded Iris controller VM from `e2-standard-4` (16GB) to
`e2-highmem-4` (32GB) in prod and dev

## Ray min_workers changes

| Cluster | Node type | Before | After |
|---|---|---|---|
| us-central1 | tpu_worker (v5p-8) | 1 | 0 |
| us-central1 | tpu_slice_v5p_8 | 12 | 6 |
| us-central1 | tpu_slice_v5p_16 | 1 | 0 |
| us-central1 | tpu_slice_v5p_32 | 1 | 0 |
| us-central1 | tpu_slice_v5p_64 | 1 | 0 |
| us-central2 | tpu_worker (v4-8) | 4 | 2 |
| us-central2-staging | tpu_worker (v4-8) | 4 | 2 |
| eu-west4 | tpu_worker (v5e-4) | 4 | 2 |
| eu-west4 | tpu_slice_v5e_128 | 1 | 0 |
| eu-west4-a | tpu_slice_v6e_128 | 2 | 1 |
| us-east5-a | tpu_worker (v5p-8) | 8 | 4 |
| us-east5-a | tpu_slice_v5p_8 | 8 | 4 |
| us-east5-a-vllm | tpu_worker | 1 | 0 |
| us-east5-a-vllm | tpu_slice_v5p_8 | 2 | 1 |
| us-east5-b-vllm | tpu_worker | 2 | 1 |
| eu-west4-vllm | tpu_worker | 2 | 1 |
| us-central1-vllm | tpu_worker | 1 | 0 |
| us-central1-vllm | tpu_slice_v5p_8 | 2 | 1 |
| us-central2-vllm | tpu_worker | 2 | 1 |
| us-east1-d-vllm | tpu_worker | 2 | 1 |
diff --git a/infra/marin-eu-west4-a.yaml b/infra/marin-eu-west4-a.yaml
@@ -194,7 +194,7 @@ available_node_types:
 
   tpu_slice_v6e_128:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v6e-128
       runtimeVersion: v2-alpha-tpuv6e
diff --git a/infra/marin-eu-west4-vllm.yaml b/infra/marin-eu-west4-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v5litepod-4
       runtimeVersion: v2-alpha-tpuv5-lite
diff --git a/infra/marin-eu-west4.yaml b/infra/marin-eu-west4.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 4
+    min_workers: 2
     node_config:
       acceleratorType: v5litepod-4
       runtimeVersion: v2-alpha-tpuv5-lite
@@ -194,7 +194,7 @@ available_node_types:
 
   tpu_slice_v5e_128:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5litepod-128
       runtimeVersion: v2-alpha-tpuv5-lite
diff --git a/infra/marin-us-central1-vllm.yaml b/infra/marin-us-central1-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -141,7 +141,7 @@ available_node_types:
 
   tpu_slice_v5p_8:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
diff --git a/infra/marin-us-central1.yaml b/infra/marin-us-central1.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -146,7 +146,7 @@ available_node_types:
 
   tpu_slice_v5p_8:
     max_workers: 1024
-    min_workers: 12
+    min_workers: 6
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -158,7 +158,7 @@ available_node_types:
 
   tpu_slice_v5p_16:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-16
       runtimeVersion: v2-alpha-tpuv5
@@ -170,7 +170,7 @@ available_node_types:
 
   tpu_slice_v5p_32:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-32
       runtimeVersion: v2-alpha-tpuv5
@@ -182,7 +182,7 @@ available_node_types:
 
   tpu_slice_v5p_64:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-64
       runtimeVersion: v2-alpha-tpuv5
diff --git a/infra/marin-us-central2-staging.yaml b/infra/marin-us-central2-staging.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 4
+    min_workers: 2
     node_config:
       acceleratorType: v4-8
       runtimeVersion: tpu-ubuntu2204-base
diff --git a/infra/marin-us-central2-vllm.yaml b/infra/marin-us-central2-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v4-8
       runtimeVersion: tpu-ubuntu2204-base
diff --git a/infra/marin-us-central2.yaml b/infra/marin-us-central2.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 4
+    min_workers: 2
     node_config:
       acceleratorType: v4-8
       runtimeVersion: tpu-ubuntu2204-base
diff --git a/infra/marin-us-east1-d-vllm.yaml b/infra/marin-us-east1-d-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v6e-8
       runtimeVersion: v2-alpha-tpuv6e
diff --git a/infra/marin-us-east5-a-vllm.yaml b/infra/marin-us-east5-a-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 1
+    min_workers: 0
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -141,7 +141,7 @@ available_node_types:
 
   tpu_slice_v5p_8:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
diff --git a/infra/marin-us-east5-a.yaml b/infra/marin-us-east5-a.yaml
@@ -134,7 +134,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 8
+    min_workers: 4
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
@@ -146,7 +146,7 @@ available_node_types:
 
   tpu_slice_v5p_8:
     max_workers: 1024
-    min_workers: 8
+    min_workers: 4
     node_config:
       acceleratorType: v5p-8
       runtimeVersion: v2-alpha-tpuv5
diff --git a/infra/marin-us-east5-b-vllm.yaml b/infra/marin-us-east5-b-vllm.yaml
@@ -129,7 +129,7 @@ available_node_types:
             sourceImage: projects/ubuntu-os-cloud/global/images/family/ubuntu-2204-lts
   tpu_worker:
     max_workers: 1024
-    min_workers: 2
+    min_workers: 1
     node_config:
       acceleratorType: v6e-8
       runtimeVersion: v2-alpha-tpuv6e
diff --git a/lib/iris/examples/marin-dev.yaml b/lib/iris/examples/marin-dev.yaml
@@ -29,7 +29,7 @@ controller:
   image: ghcr.io/marin-community/iris-controller:latest
   gcp:
     zone: us-central1-a
-    machine_type: e2-standard-4
+    machine_type: e2-highmem-4
     boot_disk_size_gb: 100
     port: 10000
 
diff --git a/lib/iris/examples/marin.yaml b/lib/iris/examples/marin.yaml
@@ -28,7 +28,7 @@ controller:
   image: ghcr.io/marin-community/iris-controller:latest
   gcp:
     zone: us-central1-a
-    machine_type: e2-standard-4
+    machine_type: e2-highmem-4
     boot_disk_size_gb: 100
     port: 10000
 
@@ -42,7 +42,7 @@ scale_groups:
     priority: 1000
     resources: { cpu: 2, ram: 16GB, disk: 100GB, device_type: cpu, preemptible: false }
     min_slices: 0
-    max_slices: 1
+    max_slices: 4
     slice_template:
       gcp:
         mode: GCP_SLICE_MODE_VM
@@ -57,7 +57,7 @@ scale_groups:
     num_vms: 1
     priority: 10
     resources: { cpu: 112, ram: 192GB, disk: 100GB, device_type: tpu, device_variant: v5litepod-4, device_count: 4, preemptible: true }
-    min_slices: 1
+    min_slices: 3
     max_slices: 1024
     slice_template:
       gcp:
@@ -107,7 +107,7 @@ scale_groups:
     num_vms: 32
     priority: 60
     resources: { cpu: 112, ram: 192GB, disk: 100GB, device_type: tpu, device_variant: v5litepod-128, device_count: 4, preemptible: true }
-    min_slices: 0
+    min_slices: 1
     max_slices: 32
     slice_template:
       gcp:
@@ -182,7 +182,7 @@ scale_groups:
     num_vms: 32
     priority: 60
     resources: { cpu: 180, ram: 720GB, disk: 100GB, device_type: tpu, device_variant: v6e-128, device_count: 4, preemptible: true }
-    min_slices: 0
+    min_slices: 1
     max_slices: 32
     slice_template:
       gcp:
@@ -208,7 +208,7 @@ scale_groups:
     num_vms: 1
     priority: 20
     resources: { cpu: 208, ram: 448GB, disk: 100GB, device_type: tpu, device_variant: v5p-8, device_count: 4, preemptible: true }
-    min_slices: 0
+    min_slices: 8
     max_slices: 2048
     slice_template:
       gcp:
@@ -303,7 +303,7 @@ scale_groups:
     num_vms: 1
     priority: 20
     resources: { cpu: 240, ram: 400GB, disk: 100GB, device_type: tpu, device_variant: v4-8, device_count: 4, preemptible: true }
-    min_slices: 0
+    min_slices: 2
     max_slices: 2048
     slice_template:
       gcp:
