operations: add per-zone config for enabling "ingester multi-az"

ldufr · ldufr · commit 45bac9f7c061 · 2026-04-13T09:35:40.000+02:00
When enabling `multi_zone_ingester_multi_az_enabled` in the step-3 of the
migration, we change the spec of the ingester-zone-a to add the `nodeAffinity`
and that cause a rollout. Since zone-b isn't restarted at this point, it
can cause a period in which a ingester shard isn't available. This is only
true when not using ingester-zone-c for the migrations.

We add new configs that allows a more fine grain control and update the
migration process to set those specs, before we stop the zone-b, preventing
a zone-a rollout when we enable `multi_zone_ingester_multi_az_enabled`.

Signed-off-by: Laurent Dufresne &lt;laurent.dufresne@grafana.com&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -375,6 +375,7 @@
 * [ENHANCEMENT] Multi-zone: Add config validation for `-querier.prefer-availability-zones` flag on querier and ruler-querier deployments. #14539
 * [ENHANCEMENT] Distributor: render the experimental `-distributor.max-active-series-per-user` flag on distributor if `$._config.limits.max_active_series_per_user` is set. #14636
 * [ENHANCEMENT] Ingester: Add `$._config.ingest_storage_set_client_rack` to pass `-ingest-storage.kafka.client-rack` when zone-aware replication is enabled. #14654
+* [ENHANCEMENT] Ingester: Add `$._config.multi_zone_ingester_multi_az_zone_(a|b|c)_enabled` to simplify migrations not using a temporary zone-c. #15000
 * [BUGFIX] Ingester: Fix `$._config.ingest_storage_ingester_autoscaling_max_owned_series_threshold` default value, to compute it based on the configured `$._config.ingester_instance_limits.max_series`. #13448
 
 ### Documentation
diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-1-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-1-generated.yaml
@@ -4374,6 +4374,14 @@ spec:
         rollout-group: ingester
     spec:
       affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                - us-east-2a
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:
@@ -4468,6 +4476,11 @@ spec:
       securityContext:
         runAsUser: 0
       terminationGracePeriodSeconds: 1200
+      tolerations:
+      - effect: NoSchedule
+        key: topology
+        operator: Equal
+        value: secondary-az
       volumes:
       - configMap:
           name: overrides
diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-1.jsonnet b/operations/mimir-tests/test-multi-az-read-path-migration-step-1.jsonnet
@@ -18,5 +18,9 @@
     multi_zone_query_frontend_routing_enabled: false,
     multi_zone_ruler_routing_enabled: false,
     multi_zone_memcached_routing_enabled: false,
+
+    // Enable multi-az config for the ingester zone-a to prevent a rollout
+    // of zone-a when `multi_zone_ingester_multi_az_enabled` is set to true.
+    multi_zone_ingester_zone_a_multi_az_enabled: true,
   },
 }
diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2-generated.yaml
@@ -4426,6 +4426,14 @@ spec:
         rollout-group: ingester
     spec:
       affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                - us-east-2a
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:
@@ -4520,6 +4528,11 @@ spec:
       securityContext:
         runAsUser: 0
       terminationGracePeriodSeconds: 1200
+      tolerations:
+      - effect: NoSchedule
+        key: topology
+        operator: Equal
+        value: secondary-az
       volumes:
       - configMap:
           name: overrides
diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2a-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2a-generated.yaml
@@ -4426,6 +4426,14 @@ spec:
         rollout-group: ingester
     spec:
       affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                - us-east-2a
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:
@@ -4520,6 +4528,11 @@ spec:
       securityContext:
         runAsUser: 0
       terminationGracePeriodSeconds: 1200
+      tolerations:
+      - effect: NoSchedule
+        key: topology
+        operator: Equal
+        value: secondary-az
       volumes:
       - configMap:
           name: overrides
diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2b-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2b-generated.yaml
@@ -4426,6 +4426,14 @@ spec:
         rollout-group: ingester
     spec:
       affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                - us-east-2a
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:
@@ -4520,6 +4528,11 @@ spec:
       securityContext:
         runAsUser: 0
       terminationGracePeriodSeconds: 1200
+      tolerations:
+      - effect: NoSchedule
+        key: topology
+        operator: Equal
+        value: secondary-az
       volumes:
       - configMap:
           name: overrides
diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2c-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2c-generated.yaml
@@ -4426,6 +4426,14 @@ spec:
         rollout-group: ingester
     spec:
       affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                - us-east-2a
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:
@@ -4520,6 +4528,11 @@ spec:
       securityContext:
         runAsUser: 0
       terminationGracePeriodSeconds: 1200
+      tolerations:
+      - effect: NoSchedule
+        key: topology
+        operator: Equal
+        value: secondary-az
       volumes:
       - configMap:
           name: overrides
diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2d-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2d-generated.yaml
@@ -4426,6 +4426,14 @@ spec:
         rollout-group: ingester
     spec:
       affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                - us-east-2a
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:
@@ -4520,6 +4528,11 @@ spec:
       securityContext:
         runAsUser: 0
       terminationGracePeriodSeconds: 1200
+      tolerations:
+      - effect: NoSchedule
+        key: topology
+        operator: Equal
+        value: secondary-az
       volumes:
       - configMap:
           name: overrides
diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2e-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2e-generated.yaml
@@ -4403,6 +4403,14 @@ spec:
         rollout-group: ingester
     spec:
       affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                - us-east-2a
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:
@@ -4497,6 +4505,11 @@ spec:
       securityContext:
         runAsUser: 0
       terminationGracePeriodSeconds: 1200
+      tolerations:
+      - effect: NoSchedule
+        key: topology
+        operator: Equal
+        value: secondary-az
       volumes:
       - configMap:
           name: overrides
diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2f-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2f-generated.yaml
@@ -4403,6 +4403,14 @@ spec:
         rollout-group: ingester
     spec:
       affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: topology.kubernetes.io/zone
+                operator: In
+                values:
+                - us-east-2a
         podAntiAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:
           - labelSelector:
@@ -4497,6 +4505,11 @@ spec:
       securityContext:
         runAsUser: 0
       terminationGracePeriodSeconds: 1200
+      tolerations:
+      - effect: NoSchedule
+        key: topology
+        operator: Equal
+        value: secondary-az
       volumes:
       - configMap:
           name: overrides
diff --git a/operations/mimir/multi-zone-ingester.libsonnet b/operations/mimir/multi-zone-ingester.libsonnet
@@ -19,6 +19,9 @@
 
     // Controls whether the multi (virtual) zone ingester should also be deployed multi-AZ.
     multi_zone_ingester_multi_az_enabled: $._config.multi_zone_read_path_multi_az_enabled,
+    multi_zone_ingester_zone_a_multi_az_enabled: self.multi_zone_ingester_multi_az_enabled,
+    multi_zone_ingester_zone_b_multi_az_enabled: self.multi_zone_ingester_multi_az_enabled,
+    multi_zone_ingester_zone_c_multi_az_enabled: self.multi_zone_ingester_multi_az_enabled,
   },
 
   local container = $.core.v1.container,
@@ -27,12 +30,12 @@
   local service = $.core.v1.service,
   local podAntiAffinity = $.apps.v1.deployment.mixin.spec.template.spec.affinity.podAntiAffinity,
 
-  local isMultiAZEnabled = $._config.multi_zone_ingester_multi_az_enabled,
-  local isZoneAEnabled = isMultiAZEnabled && std.length($._config.multi_zone_availability_zones) >= 1,
-  local isZoneBEnabled = isMultiAZEnabled && std.length($._config.multi_zone_availability_zones) >= 2,
-  local isZoneCEnabled = isMultiAZEnabled && std.length($._config.multi_zone_availability_zones) >= 3,
+  local isZoneAEnabled = $._config.multi_zone_ingester_zone_a_multi_az_enabled && std.length($._config.multi_zone_availability_zones) >= 1,
+  local isZoneBEnabled = $._config.multi_zone_ingester_zone_b_multi_az_enabled && std.length($._config.multi_zone_availability_zones) >= 2,
+  local isZoneCEnabled = $._config.multi_zone_ingester_zone_c_multi_az_enabled && std.length($._config.multi_zone_availability_zones) >= 3,
 
-  assert !isMultiAZEnabled || $._config.multi_zone_ingester_enabled : 'ingester multi-AZ deployment requires ingester multi-zone to be enabled',
+  local isMultiAZEnabledForOneZone = isZoneAEnabled || isZoneBEnabled || isZoneCEnabled,
+  assert !isMultiAZEnabledForOneZone || $._config.multi_zone_ingester_enabled : 'ingester multi-AZ deployment requires ingester multi-zone to be enabled',
   assert !$._config.multi_zone_ingester_zpdb_enabled || $._config.rollout_operator_webhooks_enabled : 'zpdb configuration requires rollout_operator_webhooks_enabled=true',
 
   //
