Skip to content

Commit d782457

Browse files
AndrewChubatiukAndrewChubatiuk
committed
distributed: added ability to setup single global balancer for reads and writes
1 parent 391eb7c commit d782457

File tree

10 files changed

+298
-175
lines changed

10 files changed

+298
-175
lines changed

‎charts/victoria-metrics-distributed/CHANGELOG.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
## Next release
22

3-
- TODO
3+
**Update note**: This release contains breaking changes. Please follow [upgrade guide](https://docs.victoriametrics.com/helm/victoria-metrics-distributed/#upgrade-to-0240)
4+
5+
- added mixed global balancer, that allows to mix write and read traffic.
46

57
## 0.23.0
68

‎charts/victoria-metrics-distributed/_index.md.gotmpl‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,64 @@ First, performing update on availability zone `zone-eu-1`:
132132

133133
Then, perform update on availability zone `zone-us-1` with the same steps1~4.
134134

135+
### Upgrade to 0.24.0
136+
137+
Starting this release first item of `*.vmauth.spec.unauthorizedUserAccessSpec.url_map` is no longer merged with default backend configuration.
138+
Custom settings for default backend should now be defined using `defaultUrlMapItem.<accessType>.<backendType>`:
139+
140+
For global read VMAuth:
141+
142+
```
143+
read:
144+
global:
145+
vmauth:
146+
spec:
147+
unauthorizedUserAccessSpec:
148+
url_map:
149+
- load_balancing_policy: first_available
150+
retry_status_codes: [500, 502, 503]
151+
```
152+
is now
153+
```
154+
defaultUrlMapItem:
155+
read:
156+
vmsingle: # if you're using VMSingle
157+
load_balancing_policy: first_available
158+
retry_status_codes: [500, 502, 503]
159+
vmcluster: # if you're using VMCluster
160+
load_balancing_policy: first_available
161+
retry_status_codes: [500, 502, 503]
162+
```
163+
164+
For global write VMAuth:
165+
166+
```
167+
write:
168+
global:
169+
vmauth:
170+
spec:
171+
unauthorizedUserAccessSpec:
172+
url_map:
173+
- load_balancing_policy: first_available
174+
retry_status_codes: [500, 502, 503]
175+
```
176+
is now
177+
```
178+
defaultUrlMapItem:
179+
write:
180+
vmagent: # if you have VMAgents queue enabled
181+
load_balancing_policy: first_available
182+
retry_status_codes: [500, 502, 503]
183+
vmsingle: # if you're using VMSingle without VMAgents enabled
184+
load_balancing_policy: first_available
185+
retry_status_codes: [500, 502, 503]
186+
vmcluster: # if you're using VMCluster without VMAgents enabled
187+
load_balancing_policy: first_available
188+
retry_status_codes: [500, 502, 503]
189+
```
190+
191+
Same is for per zone VMAuth proxy `zoneTpl.read.vmauth`.
192+
135193
### Upgrade to 0.13.0
136194

137195
Introduction of VMCluster's [`requestsLoadBalancer`](https://docs.victoriametrics.com/operator/resources/vmcluster/#requests-load-balancing) allowed to simplify distributed chart setup by removing VMAuth CRs for read and write load balancing. Some parameters are not needed anymore:

‎charts/victoria-metrics-distributed/templates/NOTES.txt‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,10 @@ Global read endpoint is available now!
1111
{{- $_ := set $ctx "appKey" (list "read" "global" "vmauth") }}
1212
You can use `{{ include "vm.url" $ctx }}/select/<accountID>/prometheus/<suffix>` as datasource of vmalert or grafana, check more supported APIs [here](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format).
1313
{{- end }}
14+
{{ if (((.Values.mixed).global).vmauth).enabled }}
15+
Global read/write endpoint is available now!
16+
{{- $_ := set $ctx "appKey" (list "mixed" "global" "vmauth") }}
17+
You can use:
18+
- `{{ include "vm.url" $ctx }}/select/<accountID>/prometheus/<suffix>` as datasource of vmalert or grafana, check more supported APIs [here](https://docs.victoriametrics.com/victoriametrics/cluster-victoriametrics/#url-format).
19+
- `{{ include "vm.url" $ctx }}/prometheus/api/v1/write` as write destination for your client like vmagent, check more supported APIs [here](https://docs.victoriametrics.com/victoriametrics/vmagent/#how-to-push-data-to-vmagent).
20+
{{- end }}

‎charts/victoria-metrics-distributed/templates/extra-vmagent.yaml‎

Lines changed: 38 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,48 @@
11
{{- if ((.Values.extra).vmagent).enabled }}
2+
{{- $urlCtx := dict "helm" . }}
3+
{{- $_ := set $urlCtx "style" "managed" }}
24
{{- $ctx := dict "helm" . "appKey" (list "extra" "vmagent" "spec") -}}
3-
{{- $ns := include "vm.namespace" $ctx }}
5+
{{- $values := .Values }}
6+
{{- $spec := (($values.extra).vmagent).spec }}
7+
{{- $zoneTpl := $values.zoneTpl }}
8+
{{- $common := .Values.common }}
9+
{{- $isMultitenant := $values.enableMultitenancy }}
10+
{{- $tenant := ternary "multitenant" "0" $isMultitenant }}
11+
12+
{{- $urls := default list }}
13+
{{- if $values.write.global.vmauth.enabled -}}
14+
{{- $write := deepCopy (($values.write).global).vmauth }}
15+
{{- $_ := set $write "spec" (mergeOverwrite (deepCopy $common.vmauth.spec) $write.spec) }}
16+
{{- $fullname := include "vm.fullname" . }}
17+
{{- $urlCtx := dict "helm" . "appKey" (list "globalWrite" "vmauth" "spec") "globalWrite" (dict "vmauth" $write) "style" "managed" "fullname" $fullname }}
18+
{{- $urls = append $urls (printf "%s/prometheus/api/v1/write" (include "vm.url" $urlCtx)) }}
19+
{{- else }}
20+
{{- range $i, $z := $values.availabilityZones }}
21+
{{- $zone := mergeOverwrite (deepCopy $zoneTpl) $z }}
22+
{{- if $zone.vmcluster.enabled }}
23+
{{- $urlSpec := mergeOverwrite (deepCopy ((($common.vmcluster).spec | default dict))) (deepCopy ($zone.vmcluster).spec | default dict) }}
24+
{{- $_ := set $zone.vmcluster "spec" $urlSpec }}
25+
{{- $_ := set $urlCtx "zone" $zone }}
26+
{{- $_ := set $urlCtx "appKey" (list "zone" "vmcluster" "spec" "vmselect") }}
27+
{{- $urls = append $urls (printf "%s/insert/%s/prometheus/api/v1/write" (include "vm.url" $urlCtx) $tenant) }}
28+
{{- else if $zone.vmsingle.enabled }}
29+
{{- $urlSpec := mergeOverwrite (deepCopy ((($common.vmsingle).spec | default dict))) (deepCopy ($zone.vmsingle).spec | default dict) }}
30+
{{- $_ := set $zone.vmsingle "spec" $urlSpec }}
31+
{{- $_ := set $urlCtx "zone" $zone }}
32+
{{- $_ := set $urlCtx "appKey" (list "zone" "vmsingle" "spec") }}
33+
{{- $urls = append $urls (printf "%s/prometheus/api/v1/write" (include "vm.url" $urlCtx)) }}
34+
{{- end }}
35+
{{- end }}
36+
{{- end -}}
37+
38+
{{- $_ := set $spec "remoteWrites" $urls }}
439
---
540
apiVersion: operator.victoriametrics.com/v1beta1
641
kind: VMAgent
742
metadata:
843
labels: {{ include "vm.labels" $ctx | nindent 4 }}
944
name: {{ include "vm.cr.fullname" $ctx }}
10-
namespace: {{ $ns }}
11-
{{- $spec := (((.Values).extra).vmagent).spec }}
12-
{{- $write := deepCopy (.Values.write).global }}
13-
{{- $_ := set $write.vmauth "spec" (mergeOverwrite (deepCopy $.Values.common.vmauth.spec) $write.vmauth.spec) }}
14-
{{- $fullname := include "vm.fullname" . }}
15-
{{- $ctx := dict "helm" . "appKey" (list "globalWrite" "vmauth" "spec") "globalWrite" $write "style" "managed" "fullname" $fullname }}
16-
{{- $remoteWrites := list (dict "url" (printf "%s/prometheus/api/v1/write" (include "vm.url" $ctx))) }}
17-
{{- $_ := set $spec "remoteWrite" (concat $remoteWrites ($spec.remoteWrite | default list)) }}
45+
namespace: {{ include "vm.namespace" $ctx }}
1846
spec: {{ tpl (toYaml $spec) . | nindent 2 }}
47+
{{- $_ := unset $urlCtx "style" -}}
1948
{{- end }}

‎charts/victoria-metrics-distributed/templates/grafana-datasource.yaml‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,16 +1,21 @@
1-
{{ if and (((.Values.read).global).vmauth).enabled (index .Values "victoria-metrics-k8s-stack" "grafana" "enabled") }}
2-
{{- $read := deepCopy (.Values.read).global }}
3-
{{- $fullname := include "vm.fullname" . }}
4-
{{- $_ := set $read.vmauth "spec" (mergeOverwrite (deepCopy $.Values.common.vmauth.spec) $read.vmauth.spec) }}
5-
{{- $ctx := dict "helm" . "appKey" (list "globalRead" "vmauth" "spec") "globalRead" $read "style" "managed" "fullname" $fullname }}
6-
{{- $url := (printf "%s/select/0/prometheus/" (include "vm.url" $ctx)) }}
7-
{{- $ns := include "vm.namespace" $ctx }}
1+
{{- $read := default dict }}
2+
{{- if (((.Values.read).global).vmauth).enabled }}
3+
{{- $read = deepCopy ((.Values.read).global).vmauth }}
4+
{{- else if (((.Values.mixed).global).vmauth).enabled }}
5+
{{- $read = deepCopy ((.Values.mixed).global).vmauth }}
6+
{{- end }}
7+
{{- $common := .Values.common.vmauth.spec }}
8+
{{ if and $read.enabled (index .Values "victoria-metrics-k8s-stack" "grafana" "enabled") }}
9+
{{- $fullname := include "vm.fullname" . }}
10+
{{- $_ := set $read "spec" (mergeOverwrite (deepCopy $common) (deepCopy $read.spec)) }}
11+
{{- $ctx := dict "helm" . "appKey" (list "globalRead" "vmauth" "spec") "globalRead" (dict "vmauth" $read) "style" "managed" "fullname" $fullname }}
12+
{{- $url := (printf "%s/select/0/prometheus/" (include "vm.url" $ctx)) }}
813
---
914
apiVersion: v1
1015
kind: ConfigMap
1116
metadata:
12-
name: {{ include "vm.fullname" . }}-grafana-ds
13-
namespace: {{ $ns }}
17+
name: {{ $fullname }}-grafana-ds
18+
namespace: {{ include "vm.namespace" $ctx }}
1419
labels: {{ include "vm.labels" . | nindent 4 }}
1520
{{ index .Values "victoria-metrics-k8s-stack" "grafana" "sidecar" "datasources" "label" }}: "1"
1621
data:

charts/victoria-metrics-distributed/templates/per-az/vmauth-read-proxy.yaml renamed to charts/victoria-metrics-distributed/templates/per-az/vmauth.yaml

Lines changed: 34 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
1-
{{- $ns := include "vm.namespace" . }}
21
{{- $ctx := dict "helm" . }}
32
{{- $urlCtx := dict "helm" . }}
4-
{{- range $i, $z := .Values.availabilityZones }}
5-
{{- $zone := mergeOverwrite (deepCopy $.Values.zoneTpl) (deepCopy $z) }}
3+
{{- $values := .Values }}
4+
{{- $common := $values.common | default dict }}
5+
{{- $zoneTpl := $values.zoneTpl }}
6+
{{- $defaultUrlMapItem := $values.defaultUrlMapItem }}
7+
{{- range $i, $z := $values.availabilityZones }}
8+
{{- $zone := mergeOverwrite (deepCopy $zoneTpl) (deepCopy $z) }}
69
{{- if $zone.read.vmauth.enabled }}
710
---
811
apiVersion: operator.victoriametrics.com/v1beta1
@@ -11,63 +14,66 @@ metadata:
1114
{{- $_ := set $ctx "appKey" (list "zone" "read" "vmauth") }}
1215
{{- if $zone.vmcluster.enabled }}
1316
{{- $urlSpec := $zone.vmcluster.spec | default dict }}
14-
{{- $_ := set $zone.vmcluster "spec" (mergeOverwrite (deepCopy $.Values.common.vmcluster.spec) $urlSpec) }}
17+
{{- $_ := set $zone.vmcluster "spec" (mergeOverwrite (deepCopy $common.vmcluster.spec) $urlSpec) }}
1518
{{- else if $zone.vmsingle.enabled }}
1619
{{- $urlSpec := $zone.vmsingle.spec | default dict }}
17-
{{- $_ := set $zone.vmsingle "spec" (mergeOverwrite (deepCopy $.Values.common.vmsingle.spec) $urlSpec) }}
20+
{{- $_ := set $zone.vmsingle "spec" (mergeOverwrite (deepCopy $common.vmsingle.spec) $urlSpec) }}
1821
{{- end }}
1922
{{- $_ := set $ctx "zone" $zone }}
2023
{{- $_ := set $urlCtx "zone" $zone }}
2124
name: {{ include "vm.cr.fullname" $ctx }}
22-
namespace: {{ $ns }}
25+
namespace: {{ include "vm.namespace" $ctx }}
2326
labels: {{ include "vm.labels" $ctx | nindent 4 }}
2427
{{- with $zone.read.vmauth.annotations }}
2528
annotations: {{ toYaml . | nindent 4 }}
2629
{{- end }}
2730

28-
{{- $spec := mergeOverwrite (deepCopy $.Values.common.vmauth.spec) (deepCopy (($zone.common).spec)) (deepCopy ((($zone.read).vmauth).spec)) }}
31+
{{- $spec := mergeOverwrite (deepCopy $common.vmauth.spec) (deepCopy (($zone.common).spec)) (deepCopy ((($zone.read).vmauth).spec)) }}
2932
{{- $_ := set $urlCtx "style" "managed" }}
30-
{{- $urls := dict "cluster" (default list) "single" (default list) }}
33+
{{- $urls := dict "vmcluster" (default list) "vmsingle" (default list) }}
3134
{{- if $zone.read.allow }}
3235
{{- if ($zone.vmcluster).enabled }}
3336
{{- $_ := set $urlCtx "appKey" (list "zone" "vmcluster" "spec" "vmselect") }}
34-
{{- $_ := set $urls "cluster" (append $urls.cluster (include "vm.url" $urlCtx)) }}
37+
{{- $_ := set $urls "vmcluster" (append $urls.vmcluster (include "vm.url" $urlCtx)) }}
3538
{{- else if ($zone.vmsingle).enabled }}
3639
{{- $_ := set $urlCtx "appKey" (list "zone" "vmsingle" "spec") }}
37-
{{- $_ := set $urls "single" (append $urls.single (include "vm.url" $urlCtx)) }}
40+
{{- $_ := set $urls "vmsingle" (append $urls.vmsingle (include "vm.url" $urlCtx)) }}
3841
{{- end }}
3942
{{- end }}
40-
{{- range $j, $c := $.Values.availabilityZones }}
41-
{{- $cross := mergeOverwrite (deepCopy $.Values.zoneTpl) (deepCopy $c) }}
43+
{{- range $j, $c := $values.availabilityZones }}
44+
{{- $cross := mergeOverwrite (deepCopy $zoneTpl) (deepCopy $c) }}
4245
{{- if and (ne $j $i) $cross.read.allow }}
4346
{{- if ($cross.vmcluster).enabled }}
4447
{{- $_ := set $urlCtx "appKey" (list "zone" "vmcluster" "spec" "vmselect") }}
45-
{{- $_ := set $cross.vmcluster "spec" (mergeOverwrite (deepCopy $.Values.common.vmcluster.spec) (deepCopy ( $cross.vmcluster.spec | default dict))) }}
48+
{{- $_ := set $cross.vmcluster "spec" (mergeOverwrite (deepCopy $common.vmcluster.spec) (deepCopy ( $cross.vmcluster.spec | default dict))) }}
4649
{{- $_ := set $urlCtx "zone" $cross }}
47-
{{- $_ := set $urls "cluster" (append $urls.cluster (include "vm.url" $urlCtx)) }}
50+
{{- $_ := set $urls "vmcluster" (append $urls.vmcluster (include "vm.url" $urlCtx)) }}
4851
{{- else if ($cross.vmsingle).enabled }}
4952
{{- $_ := set $urlCtx "appKey" (list "zone" "vmsingle" "spec") }}
50-
{{- $_ := set $cross.vmsingle "spec" (mergeOverwrite (deepCopy $.Values.common.vmsingle.spec) (deepCopy ( $cross.vmsingle.spec | default dict))) }}
53+
{{- $_ := set $cross.vmsingle "spec" (mergeOverwrite (deepCopy $common.vmsingle.spec) (deepCopy ( $cross.vmsingle.spec | default dict))) }}
5154
{{- $_ := set $urlCtx "zone" $cross }}
52-
{{- $_ := set $urls "single" (append $urls.single (include "vm.url" $urlCtx)) }}
55+
{{- $_ := set $urls "vmsingle" (append $urls.vmsingle (include "vm.url" $urlCtx)) }}
5356
{{- end }}
5457
{{- end }}
5558
{{- end }}
5659
{{- $_ := unset $urlCtx "style" }}
57-
{{- if and (empty $urls.cluster) (empty $urls.single) }}
58-
{{- fail (printf "No read backend urls defined for vmauth in AZ %s" $zone.name) -}}
59-
{{- end }}
6060

61-
{{- $paths := dict "cluster" (list "/select/.+" "/admin/tenants") "single" (list "/select/.+") }}
62-
{{- $dropPrefixParts := dict "cluster" 0 "single" 2 }}
61+
{{- $hasBackends := false }}
62+
{{- range $_, $backends := $urls }}
63+
{{- if gt (len $backends) 0 -}}
64+
{{- $hasBackends = true -}}
65+
{{- end -}}
66+
{{- end -}}
67+
{{- if not $hasBackends }}
68+
{{- fail (printf "No backend urls for vmauth in AZ %q" $zone.name) -}}
69+
{{- end -}}
70+
6371
{{- $accessSpec := $spec.unauthorizedUserAccessSpec | default dict }}
64-
{{- $urlMap := $accessSpec.url_map | default (list (default dict)) }}
65-
{{- $firstItem := index $urlMap 0 }}
66-
{{- $urlMap = slice $urlMap 1 }}
67-
{{- range $kind, $backends := $urls }}
68-
{{- if $backends }}
69-
{{- $item := dict "src_paths" (get $paths $kind) "url_prefix" $backends "drop_src_path_prefix_parts" (get $dropPrefixParts $kind) }}
70-
{{- $item = mergeOverwrite (deepCopy $item) (deepCopy $firstItem) }}
72+
{{- $urlMap := $accessSpec.url_map | default list }}
73+
{{- range $type, $backendUrls := $urls }}
74+
{{- if $backendUrls }}
75+
{{- $item := deepCopy (index $defaultUrlMapItem "read" $type) }}
76+
{{- $_ := set $item "url_prefix" $backendUrls }}
7177
{{- $urlMap = prepend $urlMap $item }}
7278
{{- end }}
7379
{{- end }}

‎charts/victoria-metrics-distributed/templates/vmauth-read.yaml‎

Lines changed: 0 additions & 70 deletions
This file was deleted.

0 commit comments

Comments
 (0)