Skip to content

Commit 70bdbe5

Browse files
f41gh7f41gh7
committed
api/vmauth: add new field useProxyProtocol
 It instructs vmauth to use proxy protocol v2 for incoming requests. It's useful for vmauth installations behind TCP load balancers. If this option is set, vmauth starts using TCP readiness and liveness probes and disables vmservice scrape creation. Unless it has internalListenPort configured (which is recommended). Related issue: #1309
1 parent 1e5cd07 commit 70bdbe5

File tree

9 files changed

+60
-36
lines changed

9 files changed

+60
-36
lines changed

‎api/operator/v1beta1/vmauth_types.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,10 @@ type VMAuthSpec struct {
120120
// related doc https://docs.victoriametrics.com/vmauth/#security
121121
// +optional
122122
InternalListenPort string `json:"internalListenPort,omitempty"`
123+
124+
// UseProxyProtocol enables proxy protocol for vmauth
125+
// https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt
126+
UseProxyProtocol bool `json:"useProxyProtocol,omitempty"`
123127
}
124128

125129
// VMAuthUnauthorizedUserAccessSpec defines unauthorized_user section configuration for vmauth

‎api/operator/v1beta1/vmextra_types.go

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -375,15 +375,6 @@ func (ss *AdditionalServiceSpec) NameOrDefault(defaultName string) string {
375375
return defaultName + "-additional-service"
376376
}
377377

378-
// MaybeEnableProxyProtocol conditionally adds proxy protocol for custom config-reloader image
379-
// useful for vmagent and vmauth
380-
func MaybeEnableProxyProtocol(args []string, extaArgs map[string]string) []string {
381-
if v, ok := extaArgs["httpListenAddr.useProxyProtocol"]; ok && v == "true" {
382-
args = append(args, "--reload-use-proxy-protocol")
383-
}
384-
return args
385-
}
386-
387378
// BuildReloadPathWithPort builds reload api path for given args
388379
func BuildReloadPathWithPort(extraArgs map[string]string, port string) string {
389380
proto := protoFromFlags(extraArgs)

‎config/crd/overlay/crd.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16074,6 +16074,11 @@ spec:
1607416074
UseDefaultResources controls resource settings
1607516075
By default, operator sets built-in resource requirements
1607616076
type: boolean
16077+
useProxyProtocol:
16078+
description: |-
16079+
UseProxyProtocol enables proxy protocol for vmauth
16080+
https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt
16081+
type: boolean
1607716082
useStrictSecurity:
1607816083
description: |-
1607916084
UseStrictSecurity enables strict security mode for component

‎docs/CHANGELOG.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,8 @@ aliases:
2020

2121
* FEATURE: [operator](https://docs.victoriametrics.com/operator): support `VM_METRICS_VERSION` and `VM_LOGS_VERSION` env variables as a source for all VM and VL related CR image versions
2222
* FEATURE: [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth/): add new field `internalListenPort` for serving internal routes. See [this issue](https://github.com/VictoriaMetrics/operator/issues/1302) and this [docs](https://docs.victoriametrics.com/vmauth/#security).
23+
* FEATURE: [vmauth](https://docs.victoriametrics.com/operator/resources/vmauth/): add new field `useProxyProtocol` for enabling [proxy protocol](https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt
24+
) for vmauth. See [this issue](https://github.com/VictoriaMetrics/operator/issues/1309).
2325
* FEATURE: [vmalertmanager](https://docs.victoriametrics.com/operator/resources/vmalertmanager): add runtime configuration validation. See [this issue](https://github.com/VictoriaMetrics/operator/issues/1299) for details.
2426
* FEATURE: [operator](https://docs.victoriametrics.com/operator): add `StatefulSet` volumeMounts name validation. See [this issue](https://github.com/VictoriaMetrics/operator/issues/1303) for details.
2527

‎docs/api.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3434,6 +3434,7 @@ _Appears in:_
34343434
| <a href="#vmauthspec-unauthorizedaccessconfig"><code id="vmauthspec-unauthorizedaccessconfig">unauthorizedAccessConfig</code></a><br/>_[UnauthorizedAccessConfigURLMap](#unauthorizedaccessconfigurlmap) array_ | UnauthorizedAccessConfig configures access for un authorized users<br /><br />Deprecated, use unauthorizedUserAccessSpec instead<br />will be removed at v1.0 release |
34353435
| <a href="#vmauthspec-unauthorizeduseraccessspec"><code id="vmauthspec-unauthorizeduseraccessspec">unauthorizedUserAccessSpec</code></a><br/>_[VMAuthUnauthorizedUserAccessSpec](#vmauthunauthorizeduseraccessspec)_ | _(Optional)_<br/>UnauthorizedUserAccessSpec defines unauthorized_user config section of vmauth config |
34363436
| <a href="#vmauthspec-usedefaultresources"><code id="vmauthspec-usedefaultresources">useDefaultResources</code></a><br/>_boolean_ | _(Optional)_<br/>UseDefaultResources controls resource settings<br />By default, operator sets built-in resource requirements |
3437+
| <a href="#vmauthspec-useproxyprotocol"><code id="vmauthspec-useproxyprotocol">useProxyProtocol</code></a><br/>_boolean_ | UseProxyProtocol enables proxy protocol for vmauth<br />https://www.haproxy.org/download/2.3/doc/proxy-protocol.txt |
34373438
| <a href="#vmauthspec-usestrictsecurity"><code id="vmauthspec-usestrictsecurity">useStrictSecurity</code></a><br/>_boolean_ | _(Optional)_<br/>UseStrictSecurity enables strict security mode for component<br />it restricts disk writes access<br />uses non-root user out of the box<br />drops not needed security permissions |
34383439
| <a href="#vmauthspec-usevmconfigreloader"><code id="vmauthspec-usevmconfigreloader">useVMConfigReloader</code></a><br/>_boolean_ | _(Optional)_<br/>UseVMConfigReloader replaces prometheus-like config-reloader<br />with vm one. It uses secrets watch instead of file watch<br />which greatly increases speed of config updates |
34393440
| <a href="#vmauthspec-usernamespaceselector"><code id="vmauthspec-usernamespaceselector">userNamespaceSelector</code></a><br/>_[LabelSelector](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.30/#labelselector-v1-meta)_ | _(Optional)_<br/>UserNamespaceSelector Namespaces to be selected for VMAuth discovery.<br />Works in combination with Selector.<br />NamespaceSelector nil - only objects at VMAuth namespace.<br />Selector nil - only objects at NamespaceSelector namespaces.<br />If both nil - behaviour controlled by selectAllByDefault |

‎docs/env.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
| --- | --- | --- | --- |
44
| VM_USECUSTOMCONFIGRELOADER | false | false | enables custom config reloader for vmauth and vmagent, it should speed-up config reloading process. |
55
| VM_CONTAINERREGISTRY | - | false | container registry name prefix, e.g. docker.io |
6-
| VM_CUSTOMCONFIGRELOADERIMAGE | victoriametrics/operator:config-reloader-v0.48.4 | false | |
6+
| VM_CUSTOMCONFIGRELOADERIMAGE | victoriametrics/operator:config-reloader-v0.56.0 | false | |
77
| VM_PSPAUTOCREATEENABLED | false | false | |
88
| VM_CONFIG_RELOADER_LIMIT_CPU | unlimited | false | defines global resource.limits.cpu for all config-reloader containers |
99
| VM_CONFIG_RELOADER_LIMIT_MEMORY | unlimited | false | defines global resource.limits.memory for all config-reloader containers |

‎internal/controller/operator/factory/vmagent/vmagent.go

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1526,9 +1526,6 @@ func buildConfigReloaderArgs(cr *vmv1beta1.VMAgent) []string {
15261526
if cr.HasAnyRelabellingConfigs() {
15271527
args = append(args, fmt.Sprintf("--%s=%s", dirsArg, vmv1beta1.RelabelingConfigDir))
15281528
}
1529-
if useVMConfigReloader {
1530-
args = vmv1beta1.MaybeEnableProxyProtocol(args, cr.Spec.ExtraArgs)
1531-
}
15321529
if len(cr.Spec.ConfigReloaderExtraArgs) > 0 {
15331530
for idx, arg := range args {
15341531
cleanArg := strings.Split(strings.TrimLeft(arg, "-"), "=")[0]

‎internal/controller/operator/factory/vmauth/vmauth.go

Lines changed: 46 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,9 @@ func CreateOrUpdateVMAuth(ctx context.Context, cr *vmv1beta1.VMAuth, rclient cli
6868
if err := createOrUpdateVMAuthIngress(ctx, rclient, cr); err != nil {
6969
return fmt.Errorf("cannot create or update ingress for vmauth: %w", err)
7070
}
71-
if !ptr.Deref(cr.Spec.DisableSelfServiceScrape, false) {
71+
if !ptr.Deref(cr.Spec.DisableSelfServiceScrape, false) &&
72+
!useProxyProtocol(cr) &&
73+
len(cr.Spec.InternalListenPort) != 0 {
7274
if err := reconcile.VMServiceScrapeForCRD(ctx, rclient, buildServiceScrape(svc, cr)); err != nil {
7375
return err
7476
}
@@ -146,6 +148,9 @@ func makeSpecForVMAuth(cr *vmv1beta1.VMAuth) (*corev1.PodTemplateSpec, error) {
146148
}
147149
args = append(args, fmt.Sprintf("-auth.config=%s", configPath))
148150

151+
if cr.Spec.UseProxyProtocol {
152+
args = append(args, "-httpListenAddr.useProxyProtocol=true")
153+
}
149154
if cr.Spec.LogLevel != "" {
150155
args = append(args, fmt.Sprintf("-loggerLevel=%s", cr.Spec.LogLevel))
151156
}
@@ -303,7 +308,7 @@ func makeSpecForVMAuth(cr *vmv1beta1.VMAuth) (*corev1.PodTemplateSpec, error) {
303308
TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError,
304309
ImagePullPolicy: cr.Spec.Image.PullPolicy,
305310
}
306-
vmauthContainer = build.Probe(vmauthContainer, cr)
311+
vmauthContainer = addVMAuthProbes(cr, vmauthContainer)
307312

308313
// move vmauth container to the 0 index
309314
operatorContainers = append([]corev1.Container{vmauthContainer}, operatorContainers...)
@@ -508,8 +513,8 @@ func buildVMAuthConfigReloaderContainer(cr *vmv1beta1.VMAuth) corev1.Container {
508513
useVMConfigReloader := ptr.Deref(cr.Spec.UseVMConfigReloader, false)
509514
if useVMConfigReloader {
510515
configReloaderArgs = append(configReloaderArgs, fmt.Sprintf("--config-secret-name=%s/%s", cr.Namespace, cr.ConfigSecretName()))
511-
if len(cr.Spec.InternalListenPort) == 0 {
512-
configReloaderArgs = vmv1beta1.MaybeEnableProxyProtocol(configReloaderArgs, cr.Spec.ExtraArgs)
516+
if len(cr.Spec.InternalListenPort) == 0 && useProxyProtocol(cr) {
517+
configReloaderArgs = append(configReloaderArgs, "--reload-use-proxy-protocol")
513518
}
514519
} else {
515520
configReloaderArgs = append(configReloaderArgs, fmt.Sprintf("--config-file=%s", path.Join(vmAuthConfigMountGz, vmAuthConfigNameGz)))
@@ -708,3 +713,40 @@ func buildServiceScrape(svc *corev1.Service, cr *vmv1beta1.VMAuth) *vmv1beta1.VM
708713
}
709714
return b
710715
}
716+
717+
func useProxyProtocol(cr *vmv1beta1.VMAuth) bool {
718+
if cr.Spec.UseProxyProtocol {
719+
return true
720+
}
721+
if v, ok := cr.Spec.ExtraArgs["httpListenAddr.useProxyProtocol"]; ok && v == "true" {
722+
return true
723+
}
724+
725+
return false
726+
}
727+
728+
func addVMAuthProbes(cr *vmv1beta1.VMAuth, vmauthContainer corev1.Container) corev1.Container {
729+
if useProxyProtocol(cr) &&
730+
len(cr.Spec.InternalListenPort) == 0 &&
731+
cr.Spec.EmbeddedProbes == nil {
732+
probePort := intstr.Parse(cr.ProbePort())
733+
cr.Spec.EmbeddedProbes = &vmv1beta1.EmbeddedProbes{
734+
ReadinessProbe: &corev1.Probe{
735+
ProbeHandler: corev1.ProbeHandler{
736+
TCPSocket: &corev1.TCPSocketAction{
737+
Port: probePort,
738+
},
739+
},
740+
},
741+
LivenessProbe: &corev1.Probe{
742+
ProbeHandler: corev1.ProbeHandler{
743+
TCPSocket: &corev1.TCPSocketAction{
744+
Port: probePort,
745+
},
746+
},
747+
},
748+
}
749+
}
750+
vmauthContainer = build.Probe(vmauthContainer, cr)
751+
return vmauthContainer
752+
}

‎test/e2e/vmauth_test.go

Lines changed: 1 addition & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -509,25 +509,7 @@ var _ = Describe("test vmauth Controller", func() {
509509
},
510510
testStep{
511511
modify: func(cr *v1beta1vm.VMAuth) {
512-
cr.Spec.ExtraArgs = map[string]string{
513-
"httpListenAddr.useProxyProtocol": "true",
514-
}
515-
cr.Spec.EmbeddedProbes = &v1beta1vm.EmbeddedProbes{
516-
LivenessProbe: &corev1.Probe{
517-
ProbeHandler: corev1.ProbeHandler{
518-
TCPSocket: &corev1.TCPSocketAction{
519-
Port: intstr.FromInt32(8427),
520-
},
521-
},
522-
},
523-
ReadinessProbe: &corev1.Probe{
524-
ProbeHandler: corev1.ProbeHandler{
525-
TCPSocket: &corev1.TCPSocketAction{
526-
Port: intstr.FromInt32(8427),
527-
},
528-
},
529-
},
530-
}
512+
cr.Spec.UseProxyProtocol = true
531513
},
532514
verify: func(cr *v1beta1vm.VMAuth) {},
533515
},

0 commit comments

Comments
 (0)