diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c03e547c4..323207252d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -375,6 +375,7 @@ * [ENHANCEMENT] Multi-zone: Add config validation for `-querier.prefer-availability-zones` flag on querier and ruler-querier deployments. #14539 * [ENHANCEMENT] Distributor: render the experimental `-distributor.max-active-series-per-user` flag on distributor if `$._config.limits.max_active_series_per_user` is set. #14636 * [ENHANCEMENT] Ingester: Add `$._config.ingest_storage_set_client_rack` to pass `-ingest-storage.kafka.client-rack` when zone-aware replication is enabled. #14654 +* [ENHANCEMENT] Ingester: Add `$._config.multi_zone_ingester_multi_az_zone_(a|b|c)_enabled` to simplify migrations not using a temporary zone-c. #15000 * [BUGFIX] Ingester: Fix `$._config.ingest_storage_ingester_autoscaling_max_owned_series_threshold` default value, to compute it based on the configured `$._config.ingester_instance_limits.max_series`. #13448 ### Documentation diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-1-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-1-generated.yaml index a1792403e0..e0a0fc43e0 100644 --- a/operations/mimir-tests/test-multi-az-read-path-migration-step-1-generated.yaml +++ b/operations/mimir-tests/test-multi-az-read-path-migration-step-1-generated.yaml @@ -4374,6 +4374,14 @@ spec: rollout-group: ingester spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - us-east-2a podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -4468,6 +4476,11 @@ spec: securityContext: runAsUser: 0 terminationGracePeriodSeconds: 1200 + tolerations: + - effect: NoSchedule + key: topology + operator: Equal + value: secondary-az volumes: - configMap: name: overrides diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-1.jsonnet b/operations/mimir-tests/test-multi-az-read-path-migration-step-1.jsonnet index a33fa289a4..871fdac0f0 100644 --- a/operations/mimir-tests/test-multi-az-read-path-migration-step-1.jsonnet +++ b/operations/mimir-tests/test-multi-az-read-path-migration-step-1.jsonnet @@ -18,5 +18,9 @@ multi_zone_query_frontend_routing_enabled: false, multi_zone_ruler_routing_enabled: false, multi_zone_memcached_routing_enabled: false, + + // Enable multi-az config for the ingester zone-a to prevent a rollout + // of zone-a when `multi_zone_ingester_multi_az_enabled` is set to true. + multi_zone_ingester_zone_a_multi_az_enabled: true, }, } diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2-generated.yaml index ac7e52f9b5..540336681a 100644 --- a/operations/mimir-tests/test-multi-az-read-path-migration-step-2-generated.yaml +++ b/operations/mimir-tests/test-multi-az-read-path-migration-step-2-generated.yaml @@ -4426,6 +4426,14 @@ spec: rollout-group: ingester spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - us-east-2a podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -4520,6 +4528,11 @@ spec: securityContext: runAsUser: 0 terminationGracePeriodSeconds: 1200 + tolerations: + - effect: NoSchedule + key: topology + operator: Equal + value: secondary-az volumes: - configMap: name: overrides diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2a-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2a-generated.yaml index ac7e52f9b5..540336681a 100644 --- a/operations/mimir-tests/test-multi-az-read-path-migration-step-2a-generated.yaml +++ b/operations/mimir-tests/test-multi-az-read-path-migration-step-2a-generated.yaml @@ -4426,6 +4426,14 @@ spec: rollout-group: ingester spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - us-east-2a podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -4520,6 +4528,11 @@ spec: securityContext: runAsUser: 0 terminationGracePeriodSeconds: 1200 + tolerations: + - effect: NoSchedule + key: topology + operator: Equal + value: secondary-az volumes: - configMap: name: overrides diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2b-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2b-generated.yaml index d4876deeea..395eea1af1 100644 --- a/operations/mimir-tests/test-multi-az-read-path-migration-step-2b-generated.yaml +++ b/operations/mimir-tests/test-multi-az-read-path-migration-step-2b-generated.yaml @@ -4426,6 +4426,14 @@ spec: rollout-group: ingester spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - us-east-2a podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -4520,6 +4528,11 @@ spec: securityContext: runAsUser: 0 terminationGracePeriodSeconds: 1200 + tolerations: + - effect: NoSchedule + key: topology + operator: Equal + value: secondary-az volumes: - configMap: name: overrides diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2c-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2c-generated.yaml index c84de7976d..f20a9c2c59 100644 --- a/operations/mimir-tests/test-multi-az-read-path-migration-step-2c-generated.yaml +++ b/operations/mimir-tests/test-multi-az-read-path-migration-step-2c-generated.yaml @@ -4426,6 +4426,14 @@ spec: rollout-group: ingester spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - us-east-2a podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -4520,6 +4528,11 @@ spec: securityContext: runAsUser: 0 terminationGracePeriodSeconds: 1200 + tolerations: + - effect: NoSchedule + key: topology + operator: Equal + value: secondary-az volumes: - configMap: name: overrides diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2d-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2d-generated.yaml index b9113cce04..04d79e6095 100644 --- a/operations/mimir-tests/test-multi-az-read-path-migration-step-2d-generated.yaml +++ b/operations/mimir-tests/test-multi-az-read-path-migration-step-2d-generated.yaml @@ -4426,6 +4426,14 @@ spec: rollout-group: ingester spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - us-east-2a podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -4520,6 +4528,11 @@ spec: securityContext: runAsUser: 0 terminationGracePeriodSeconds: 1200 + tolerations: + - effect: NoSchedule + key: topology + operator: Equal + value: secondary-az volumes: - configMap: name: overrides diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2e-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2e-generated.yaml index 591d966db6..ac4e4642e0 100644 --- a/operations/mimir-tests/test-multi-az-read-path-migration-step-2e-generated.yaml +++ b/operations/mimir-tests/test-multi-az-read-path-migration-step-2e-generated.yaml @@ -4403,6 +4403,14 @@ spec: rollout-group: ingester spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - us-east-2a podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -4497,6 +4505,11 @@ spec: securityContext: runAsUser: 0 terminationGracePeriodSeconds: 1200 + tolerations: + - effect: NoSchedule + key: topology + operator: Equal + value: secondary-az volumes: - configMap: name: overrides diff --git a/operations/mimir-tests/test-multi-az-read-path-migration-step-2f-generated.yaml b/operations/mimir-tests/test-multi-az-read-path-migration-step-2f-generated.yaml index 650515346d..6952a7ac22 100644 --- a/operations/mimir-tests/test-multi-az-read-path-migration-step-2f-generated.yaml +++ b/operations/mimir-tests/test-multi-az-read-path-migration-step-2f-generated.yaml @@ -4403,6 +4403,14 @@ spec: rollout-group: ingester spec: affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: topology.kubernetes.io/zone + operator: In + values: + - us-east-2a podAntiAffinity: requiredDuringSchedulingIgnoredDuringExecution: - labelSelector: @@ -4497,6 +4505,11 @@ spec: securityContext: runAsUser: 0 terminationGracePeriodSeconds: 1200 + tolerations: + - effect: NoSchedule + key: topology + operator: Equal + value: secondary-az volumes: - configMap: name: overrides diff --git a/operations/mimir/multi-zone-ingester.libsonnet b/operations/mimir/multi-zone-ingester.libsonnet index 850859bdb7..60f79a0f7d 100644 --- a/operations/mimir/multi-zone-ingester.libsonnet +++ b/operations/mimir/multi-zone-ingester.libsonnet @@ -19,6 +19,9 @@ // Controls whether the multi (virtual) zone ingester should also be deployed multi-AZ. multi_zone_ingester_multi_az_enabled: $._config.multi_zone_read_path_multi_az_enabled, + multi_zone_ingester_zone_a_multi_az_enabled: self.multi_zone_ingester_multi_az_enabled, + multi_zone_ingester_zone_b_multi_az_enabled: self.multi_zone_ingester_multi_az_enabled, + multi_zone_ingester_zone_c_multi_az_enabled: self.multi_zone_ingester_multi_az_enabled, }, local container = $.core.v1.container, @@ -27,12 +30,12 @@ local service = $.core.v1.service, local podAntiAffinity = $.apps.v1.deployment.mixin.spec.template.spec.affinity.podAntiAffinity, - local isMultiAZEnabled = $._config.multi_zone_ingester_multi_az_enabled, - local isZoneAEnabled = isMultiAZEnabled && std.length($._config.multi_zone_availability_zones) >= 1, - local isZoneBEnabled = isMultiAZEnabled && std.length($._config.multi_zone_availability_zones) >= 2, - local isZoneCEnabled = isMultiAZEnabled && std.length($._config.multi_zone_availability_zones) >= 3, + local isZoneAEnabled = $._config.multi_zone_ingester_zone_a_multi_az_enabled && std.length($._config.multi_zone_availability_zones) >= 1, + local isZoneBEnabled = $._config.multi_zone_ingester_zone_b_multi_az_enabled && std.length($._config.multi_zone_availability_zones) >= 2, + local isZoneCEnabled = $._config.multi_zone_ingester_zone_c_multi_az_enabled && std.length($._config.multi_zone_availability_zones) >= 3, - assert !isMultiAZEnabled || $._config.multi_zone_ingester_enabled : 'ingester multi-AZ deployment requires ingester multi-zone to be enabled', + local isMultiAZAtLeastOnceEnabled = isZoneAEnabled || isZoneBEnabled || isZoneCEnabled, + assert !isMultiAZAtLeastOnceEnabled || $._config.multi_zone_ingester_enabled : 'ingester multi-AZ deployment requires ingester multi-zone to be enabled', assert !$._config.multi_zone_ingester_zpdb_enabled || $._config.rollout_operator_webhooks_enabled : 'zpdb configuration requires rollout_operator_webhooks_enabled=true', //