Skip to content

Commit 965a060

Browse files
committed
feat(controller): support hostNetwork
This commit adds the support to enable host network in the Kubewarden stack. This is done by enabling a CLI flag in the controller. Once this is done, all the policy server deployments are configured to use host network as well. Furthermore, to allow user to fix port conflicts issues, 3 new fields have been added to the policy server spec. This fields allow users to define the ports to be used by the policy server deployment. Assisted-by: Github Copilot Signed-off-by: José Guilherme Vanz <jguilhermevanz@suse.com> refactor(webhook): remove metricsPort conflict validation spec.metricsPort only controls the metrics Service Port (the externally visible scrape port) and has no effect on pod-side ports. Since it operates at a different layer than webhookPort and readinessProbePort, there is no meaningful conflict to validate against. The only pod-side conflict check that remains is webhookPort vs readinessProbePort. As a consequence, the defaultMetricsPort parameter is removed from SetupWebhookWithManager and the policyServerValidator struct, simplifying the webhook setup chain in main.go. Assisted-by: Github Copilot Signed-off-by: José Guilherme Vanz <jguilhermevanz@suse.com>
1 parent 92bcf07 commit 965a060

29 files changed

Lines changed: 1227 additions & 113 deletions

Makefile

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,9 @@ helm-unittest:
5454
test-e2e: controller-image audit-scanner-image policy-server-image
5555
$(GO_BUILD_ENV) go test ./e2e/ -v
5656

57+
.PHONY: test-all
58+
test-all: test helm-unittest test-e2e
59+
5760
.PHONY: fmt-go
5861
fmt-go:
5962
$(GO_BUILD_ENV) go fmt ./...

api/policies/v1/factories.go

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -487,6 +487,9 @@ type PolicyServerBuilder struct {
487487
requests corev1.ResourceList
488488
sigstoreTrustConfigMap string
489489
namespacedPoliciesCapabilities []string
490+
webhookPort *int32
491+
readinessProbePort *int32
492+
metricsPort *int32
490493
}
491494

492495
func NewPolicyServerFactory() *PolicyServerBuilder {
@@ -535,6 +538,21 @@ func (f *PolicyServerBuilder) WithNamespacedPoliciesCapabilities(capabilities []
535538
return f
536539
}
537540

541+
func (f *PolicyServerBuilder) WithWebhookPort(port int32) *PolicyServerBuilder {
542+
f.webhookPort = &port
543+
return f
544+
}
545+
546+
func (f *PolicyServerBuilder) WithReadinessProbePort(port int32) *PolicyServerBuilder {
547+
f.readinessProbePort = &port
548+
return f
549+
}
550+
551+
func (f *PolicyServerBuilder) WithMetricsPort(port int32) *PolicyServerBuilder {
552+
f.metricsPort = &port
553+
return f
554+
}
555+
538556
func (f *PolicyServerBuilder) Build() *PolicyServer {
539557
policyServer := PolicyServer{
540558
ObjectMeta: metav1.ObjectMeta{
@@ -559,6 +577,9 @@ func (f *PolicyServerBuilder) Build() *PolicyServer {
559577
Requests: f.requests,
560578
SigstoreTrustConfig: f.sigstoreTrustConfigMap,
561579
NamespacedPoliciesCapabilities: f.namespacedPoliciesCapabilities,
580+
WebhookPort: f.webhookPort,
581+
ReadinessProbePort: f.readinessProbePort,
582+
MetricsPort: f.metricsPort,
562583
Env: []corev1.EnvVar{
563584
{
564585
Name: "KUBEWARDEN_LOG_LEVEL",

api/policies/v1/policyserver_types.go

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,6 +150,44 @@ type PolicyServerSpec struct {
150150
// - Specific capability paths (e.g. "oci/v1/verify", "net/v1/dns_lookup_host")
151151
// +optional
152152
NamespacedPoliciesCapabilities []string `json:"namespacedPoliciesCapabilities,omitempty"`
153+
154+
// Port where the policy server listens for incoming webhook requests.
155+
// When unset, defaults to 8443. This is the port the Kubernetes API server
156+
// reaches when evaluating admission requests.
157+
// +optional
158+
// +kubebuilder:validation:Minimum=1
159+
// +kubebuilder:validation:Maximum=65535
160+
WebhookPort *int32 `json:"webhookPort,omitempty"`
161+
162+
// Port used by the policy server to expose the readiness probe endpoint.
163+
// When unset, defaults to 8081.
164+
// +optional
165+
// +kubebuilder:validation:Minimum=1
166+
// +kubebuilder:validation:Maximum=65535
167+
ReadinessProbePort *int32 `json:"readinessProbePort,omitempty"`
168+
169+
// Port exposed by the metrics Service for this policy server.
170+
// When unset, defaults to the controller-wide default
171+
// (KUBEWARDEN_POLICY_SERVER_SERVICES_METRICS_PORT env var, or 8080).
172+
// Only relevant when metrics are enabled.
173+
//
174+
// Use this field to customize which port Prometheus scrapes for this
175+
// PolicyServer's metrics Service (e.g. to match naming conventions or
176+
// avoid Service-level port collisions).
177+
//
178+
// NOTE: this field controls only the Service Port (the externally visible
179+
// scrape port). The Service TargetPort — the port the pod actually listens
180+
// on — is always the controller-wide default and is not affected by this
181+
// field. This is intentional: when the OpenTelemetry sidecar mode is
182+
// enabled, each pod gets its own injected sidecar, but the pod-side
183+
// Prometheus listener port is determined by controller-wide/injection
184+
// configuration, not per PolicyServer. Therefore, changing this field does
185+
// not change the pod listener port and will not resolve pod-port conflicts
186+
// such as those caused by hostNetwork.
187+
// +optional
188+
// +kubebuilder:validation:Minimum=1
189+
// +kubebuilder:validation:Maximum=65535
190+
MetricsPort *int32 `json:"metricsPort,omitempty"`
153191
}
154192

155193
type ReconciliationTransitionReason string
@@ -222,6 +260,34 @@ func (ps *PolicyServer) AppLabel() string {
222260
return "kubewarden-" + ps.NameWithPrefix()
223261
}
224262

263+
// EffectiveWebhookPort returns the port the policy server listens on for
264+
// admission webhook requests, using the CRD field when set or the default constant.
265+
func (ps *PolicyServer) EffectiveWebhookPort() int32 {
266+
if ps.Spec.WebhookPort != nil {
267+
return *ps.Spec.WebhookPort
268+
}
269+
return constants.PolicyServerListenPort
270+
}
271+
272+
// EffectiveReadinessProbePort returns the port used for the readiness probe,
273+
// using the CRD field when set or the default constant.
274+
func (ps *PolicyServer) EffectiveReadinessProbePort() int32 {
275+
if ps.Spec.ReadinessProbePort != nil {
276+
return *ps.Spec.ReadinessProbePort
277+
}
278+
return constants.PolicyServerReadinessProbePort
279+
}
280+
281+
// EffectiveMetricsPort returns the port used to expose the metrics endpoint.
282+
// It returns the CRD-level override when set, otherwise it falls back to defaultPort
283+
// (which is typically the controller-wide default configured via environment variable).
284+
func (ps *PolicyServer) EffectiveMetricsPort(defaultPort int32) int32 {
285+
if ps.Spec.MetricsPort != nil {
286+
return *ps.Spec.MetricsPort
287+
}
288+
return defaultPort
289+
}
290+
225291
// CommonLabels returns the common labels to be used with the resources
226292
// associated to a Policy Server. The labels defined follow
227293
// Kubernetes guidelines: https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels

api/policies/v1/policyserver_webhook.go

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -175,6 +175,7 @@ func (v *policyServerValidator) validate(ctx context.Context, policyServer *Poli
175175

176176
allErrs = append(allErrs, validateLimitsAndRequests(policyServer.Spec.Limits, policyServer.Spec.Requests)...)
177177
allErrs = append(allErrs, validateNamespacedPoliciesCapabilities(policyServer.Spec.NamespacedPoliciesCapabilities)...)
178+
allErrs = append(allErrs, v.validatePorts(policyServer)...)
178179

179180
if len(allErrs) == 0 {
180181
return nil
@@ -274,7 +275,6 @@ func validateNamespacedPoliciesCapabilities(capabilities []string) field.ErrorLi
274275
allErrs = append(allErrs, err)
275276
}
276277
}
277-
278278
return allErrs
279279
}
280280

@@ -327,3 +327,24 @@ func validateSingleCapability(pattern string, path *field.Path) *field.Error {
327327
fmt.Sprintf("%q is not a complete capability path; use %q to allow all capabilities under it",
328328
pattern, pattern+"/*"))
329329
}
330+
331+
// validatePorts checks that the port fields in the PolicyServer spec do not
332+
// conflict with each other. Only pod-side ports (webhookPort, readinessProbePort)
333+
// are validated against each other. spec.metricsPort is a Service-layer-only
334+
// setting and cannot conflict with pod-side ports.
335+
func (v *policyServerValidator) validatePorts(policyServer *PolicyServer) field.ErrorList {
336+
var allErrs field.ErrorList
337+
338+
webhookPort := policyServer.EffectiveWebhookPort()
339+
readinessPort := policyServer.EffectiveReadinessProbePort()
340+
341+
if webhookPort == readinessPort {
342+
allErrs = append(allErrs, field.Invalid(
343+
field.NewPath("spec").Child("readinessProbePort"),
344+
readinessPort,
345+
fmt.Sprintf("readinessProbePort must differ from webhookPort (%d)", webhookPort),
346+
))
347+
}
348+
349+
return allErrs
350+
}

api/policies/v1/policyserver_webhook_test.go

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -412,3 +412,76 @@ func TestPolicyServerValidateNamespacedPoliciesCapabilities(t *testing.T) {
412412
})
413413
}
414414
}
415+
416+
func TestValidatePorts(t *testing.T) {
417+
tests := []struct {
418+
name string
419+
webhookPort *int32
420+
readiness *int32
421+
metrics *int32
422+
errContains string
423+
}{
424+
{
425+
name: "all defaults, no conflict",
426+
errContains: "",
427+
},
428+
{
429+
name: "webhookPort equals readinessProbePort",
430+
webhookPort: ptr.To[int32](8081),
431+
readiness: ptr.To[int32](8081),
432+
errContains: "readinessProbePort must differ from webhookPort",
433+
},
434+
{
435+
name: "webhookPort and readinessProbePort distinct custom values",
436+
webhookPort: ptr.To[int32](9443),
437+
readiness: ptr.To[int32](9081),
438+
errContains: "",
439+
},
440+
{
441+
// metricsPort is a Service-layer-only setting and cannot conflict
442+
// with pod-side ports (webhookPort, readinessProbePort).
443+
name: "metricsPort equal to webhookPort is allowed",
444+
webhookPort: ptr.To[int32](8080),
445+
metrics: ptr.To[int32](8080),
446+
errContains: "",
447+
},
448+
{
449+
// metricsPort is a Service-layer-only setting and cannot conflict
450+
// with pod-side ports (webhookPort, readinessProbePort).
451+
name: "metricsPort equal to readinessProbePort is allowed",
452+
readiness: ptr.To[int32](9000),
453+
metrics: ptr.To[int32](9000),
454+
errContains: "",
455+
},
456+
}
457+
458+
for _, test := range tests {
459+
t.Run(test.name, func(t *testing.T) {
460+
k8sClient := fake.NewClientBuilder().Build()
461+
builder := NewPolicyServerFactory()
462+
if test.webhookPort != nil {
463+
builder = builder.WithWebhookPort(*test.webhookPort)
464+
}
465+
if test.readiness != nil {
466+
builder = builder.WithReadinessProbePort(*test.readiness)
467+
}
468+
if test.metrics != nil {
469+
builder = builder.WithMetricsPort(*test.metrics)
470+
}
471+
policyServer := builder.Build()
472+
473+
validator := policyServerValidator{
474+
deploymentsNamespace: "default",
475+
k8sClient: k8sClient,
476+
logger: logr.Discard(),
477+
}
478+
err := validator.validate(t.Context(), policyServer)
479+
480+
if test.errContains != "" {
481+
require.ErrorContains(t, err, test.errContains)
482+
} else {
483+
require.NoError(t, err)
484+
}
485+
})
486+
}
487+
}

api/policies/v1/zz_generated.deepcopy.go

Lines changed: 30 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

charts/kubewarden-controller/templates/NOTES.txt

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,11 @@ If you'd like to support us, we'd love to hear from you as one of our adopters.
1111
Adopters can be public or private.
1212

1313
Learn how to add your organization as a Kubewarden adopter by checking out the ADOPTERS.md file here: https://github.com/kubewarden/community/blob/main/ADOPTERS.md
14+
15+
{{ if .Values.hostNetwork }}
16+
⚠️ WARNING ⚠️
17+
Host Network is enabled. Ensure you set appropriate podAntiAffinity rules to prevent host port conflicts between controller replicas on the same node.
18+
{{ if eq .Values.telemetry.mode "sidecar" }}
19+
Telemetry sidecar mode (telemetry.mode=sidecar) is incompatible with host network. This chart rejects that configuration and rendering/installation will fail when hostNetwork=true and telemetry.mode=sidecar. Use telemetry.mode=custom with a remote collector instead, or disable hostNetwork.
20+
{{- end }}
21+
{{- end }}

charts/kubewarden-controller/templates/_helpers.tpl

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -181,3 +181,31 @@ are configured.
181181
- {{ .Values.auditScanner.reportCRDsKind }}
182182
{{- end -}}
183183
{{- end -}}
184+
185+
{{/*
186+
Compute the effective affinity for the controller deployment.
187+
Uses the controller-specific affinity if set, otherwise falls back to
188+
global.affinity for backward compatibility.
189+
190+
NOTE: When hostNetwork is enabled, users are responsible for setting
191+
appropriate podAntiAffinity rules to prevent host-port conflicts between
192+
controller replicas on the same node.
193+
*/}}
194+
{{- define "kubewarden-controller.effectiveAffinity" -}}
195+
{{- if .Values.affinity -}}
196+
{{- toYaml .Values.affinity -}}
197+
{{- else if .Values.global.affinity -}}
198+
{{- toYaml .Values.global.affinity -}}
199+
{{- end -}}
200+
{{- end -}}
201+
202+
{{/*
203+
Validate that hostNetwork and telemetry sidecar mode are not both enabled.
204+
They are incompatible because multiple OTel sidecars on the same node would
205+
cause port conflicts in host-network mode.
206+
*/}}
207+
{{- define "kubewarden-controller.validateHostNetworkSidecar" -}}
208+
{{- if and .Values.hostNetwork (eq .Values.telemetry.mode "sidecar") (or .Values.telemetry.metrics .Values.telemetry.tracing) -}}
209+
{{- fail "hostNetwork and telemetry.mode=sidecar are incompatible: OpenTelemetry sidecar injection causes port conflicts in host-network mode. Use telemetry.mode=custom with a remote collector instead." -}}
210+
{{- end -}}
211+
{{- end -}}

0 commit comments

Comments
 (0)