From 679c049c6a14d6a7cf44969cf6296d2d859f0a1d Mon Sep 17 00:00:00 2001 From: Joel Teo Date: Fri, 17 Apr 2026 11:16:47 +0800 Subject: [PATCH 1/4] Add downward-node-labels propagation to Kibana, Logstash and Elastic Agent CRDs Mirrors the existing Elasticsearch behavior: users can set the `eck.k8s.elastic.co/downward-node-labels` annotation on the CR with a comma-separated list of node labels, and the operator copies those labels as annotations on the managed Pods. Pods can then read them via the Downward API. The parser is extracted to a pure-Go `pkg/utils/nodelabels` package to avoid an apis->controllers import cycle, and a shared `pkg/controller/common/nodelabels` helper exposes `AnnotatePods` for the three new controllers to reuse. --- pkg/apis/agent/v1alpha1/agent_types.go | 15 ++ pkg/apis/kibana/v1/kibana_types.go | 15 ++ pkg/apis/logstash/v1alpha1/logstash_types.go | 16 +++ pkg/controller/agent/driver.go | 17 ++- pkg/controller/agent/pod.go | 6 + .../common/nodelabels/nodelabels.go | 132 ++++++++++++++++++ .../common/nodelabels/nodelabels_test.go | 131 +++++++++++++++++ pkg/controller/kibana/driver.go | 18 +++ pkg/controller/logstash/driver.go | 16 ++- pkg/controller/logstash/pod.go | 6 + pkg/utils/nodelabels/nodelabels.go | 40 ++++++ pkg/utils/nodelabels/nodelabels_test.go | 42 ++++++ 12 files changed, 452 insertions(+), 2 deletions(-) create mode 100644 pkg/controller/common/nodelabels/nodelabels.go create mode 100644 pkg/controller/common/nodelabels/nodelabels_test.go create mode 100644 pkg/utils/nodelabels/nodelabels.go create mode 100644 pkg/utils/nodelabels/nodelabels_test.go diff --git a/pkg/apis/agent/v1alpha1/agent_types.go b/pkg/apis/agent/v1alpha1/agent_types.go index f1cd2e06502..fb504e6a24c 100644 --- a/pkg/apis/agent/v1alpha1/agent_types.go +++ b/pkg/apis/agent/v1alpha1/agent_types.go @@ -14,6 +14,7 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -22,6 +23,9 @@ const ( Kind = "Agent" // FleetServerServiceAccount is the Elasticsearch service account to be used to authenticate. FleetServerServiceAccount commonv1.ServiceAccountName = "fleet-server" + // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels + // to be set as annotations on the Elastic Agent Pods. + DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // AgentSpec defines the desired state of the Agent @@ -304,6 +308,17 @@ func (a *Agent) IsMarkedForDeletion() bool { return !a.DeletionTimestamp.IsZero() } +// DownwardNodeLabels returns the node labels to copy as annotations on the Agent Pods, +// as declared via the DownwardNodeLabelsAnnotation annotation. +func (a *Agent) DownwardNodeLabels() []string { + return nodelabels.FromAnnotations(a.Annotations) +} + +// HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Agent Pods. +func (a *Agent) HasDownwardNodeLabels() bool { + return len(a.DownwardNodeLabels()) > 0 +} + func (a *Agent) AssociationStatusMap(typ commonv1.AssociationType) commonv1.AssociationStatusMap { switch typ { case commonv1.ElasticsearchAssociationType: diff --git a/pkg/apis/kibana/v1/kibana_types.go b/pkg/apis/kibana/v1/kibana_types.go index bdf40d6048c..68a7f83d51d 100644 --- a/pkg/apis/kibana/v1/kibana_types.go +++ b/pkg/apis/kibana/v1/kibana_types.go @@ -13,6 +13,7 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -22,6 +23,9 @@ const ( Kind = "Kibana" // KibanaServiceAccount is the Elasticsearch service account to be used to authenticate. KibanaServiceAccount commonv1.ServiceAccountName = "kibana" + // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels + // to be set as annotations on the Kibana Pods. + DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // +kubebuilder:object:root=true @@ -148,6 +152,17 @@ func (k *Kibana) IsMarkedForDeletion() bool { return !k.DeletionTimestamp.IsZero() } +// DownwardNodeLabels returns the node labels to copy as annotations on the Kibana Pods, +// as declared via the DownwardNodeLabelsAnnotation annotation. +func (k *Kibana) DownwardNodeLabels() []string { + return nodelabels.FromAnnotations(k.Annotations) +} + +// HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Kibana Pods. +func (k *Kibana) HasDownwardNodeLabels() bool { + return len(k.DownwardNodeLabels()) > 0 +} + func (k *Kibana) SecureSettings() []commonv1.SecretSource { return k.Spec.SecureSettings } diff --git a/pkg/apis/logstash/v1alpha1/logstash_types.go b/pkg/apis/logstash/v1alpha1/logstash_types.go index ac2ed40971d..e1911872fc8 100644 --- a/pkg/apis/logstash/v1alpha1/logstash_types.go +++ b/pkg/apis/logstash/v1alpha1/logstash_types.go @@ -13,6 +13,7 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/hash" + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) type LogstashHealth string @@ -35,6 +36,10 @@ const ( // 1) all Pods are Ready, and // 2) any associations are configured and established LogstashGreenHealth LogstashHealth = "green" + + // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels + // to be set as annotations on the Logstash Pods. + DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // LogstashSpec defines the desired state of Logstash @@ -214,6 +219,17 @@ func (l *Logstash) IsMarkedForDeletion() bool { return !l.DeletionTimestamp.IsZero() } +// DownwardNodeLabels returns the node labels to copy as annotations on the Logstash Pods, +// as declared via the DownwardNodeLabelsAnnotation annotation. +func (l *Logstash) DownwardNodeLabels() []string { + return nodelabels.FromAnnotations(l.Annotations) +} + +// HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Logstash Pods. +func (l *Logstash) HasDownwardNodeLabels() bool { + return len(l.DownwardNodeLabels()) > 0 +} + // GetObservedGeneration will return the observedGeneration from the Elastic Logstash's status. func (l *Logstash) GetObservedGeneration() int64 { return l.Status.ObservedGeneration diff --git a/pkg/controller/agent/driver.go b/pkg/controller/agent/driver.go index fa0212889bc..f795797659d 100644 --- a/pkg/controller/agent/driver.go +++ b/pkg/controller/agent/driver.go @@ -23,6 +23,7 @@ import ( commonassociation "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/association" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/defaults" + "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" @@ -153,7 +154,21 @@ func internalReconcile(params Params) (*reconciler.Results, agentv1alpha1.AgentS if err != nil { return results.WithError(err), params.Status } - return reconcilePodVehicle(params, podTemplate) + podVehicleResults, status := reconcilePodVehicle(params, podTemplate) + results.WithResults(podVehicleResults) + + // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the + // reconciliation loop as we don't want to prevent other updates from being applied. + results.WithResults(nodelabels.AnnotatePods( + params.Context, + params.Client, + params.Agent.Namespace, + map[string]string{NameLabelName: params.Agent.Name}, + params.Agent.DownwardNodeLabels(), + params.Agent.Name, + )) + + return results, status } func reconcileService(params Params) (*corev1.Service, error) { diff --git a/pkg/controller/agent/pod.go b/pkg/controller/agent/pod.go index 9006c52dcdd..279b6dc2734 100644 --- a/pkg/controller/agent/pod.go +++ b/pkg/controller/agent/pod.go @@ -178,6 +178,12 @@ func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret } vols = append(vols, caAssocVols...) + // Changes to the downward-node-labels annotation must roll the Agent Pods so the new annotations + // are re-applied on scheduling. + if params.Agent.HasDownwardNodeLabels() { + _, _ = configHash.Write([]byte(params.Agent.Annotations[agentv1alpha1.DownwardNodeLabelsAnnotation])) + } + podMeta := params.Meta.Merge(metadata.Metadata{ Labels: map[string]string{VersionLabelName: spec.Version}, Annotations: map[string]string{ConfigHashAnnotationName: fmt.Sprint(configHash.Sum32())}, diff --git a/pkg/controller/common/nodelabels/nodelabels.go b/pkg/controller/common/nodelabels/nodelabels.go new file mode 100644 index 00000000000..84ac6d31111 --- /dev/null +++ b/pkg/controller/common/nodelabels/nodelabels.go @@ -0,0 +1,132 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +// Package nodelabels contains shared helpers to propagate Kubernetes node labels to the annotations +// of Pods managed by ECK. Labels are opt-in via the DownwardNodeLabelsAnnotation on the owning +// custom resource and are copied to the Pod annotations of the resource they are set on. +package nodelabels + +import ( + "context" + "encoding/json" + "fmt" + "strings" + + "go.elastic.co/apm/v2" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" + "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s" + ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" +) + +// DownwardNodeLabelsAnnotation is re-exported for convenience. +const DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation + +// AnnotatePods copies the expected node labels as annotations on all Pods in the given namespace +// matching the given labelSelector. Missing node labels are reported as errors but do not stop the +// reconciliation of other Pods. +func AnnotatePods( + ctx context.Context, + c k8s.Client, + namespace string, + podLabelSelector map[string]string, + expectedLabels []string, + resourceName string, +) *reconciler.Results { + span, ctx := apm.StartSpan(ctx, "annotate_pods_with_node_labels", tracing.SpanTypeApp) + defer span.End() + results := reconciler.NewResult(ctx) + if len(expectedLabels) == 0 { + return results + } + pods, err := k8s.PodsMatchingLabels(c, namespace, podLabelSelector) + if err != nil { + return results.WithError(err) + } + for _, pod := range pods { + results.WithError(annotatePod(ctx, c, pod, expectedLabels, resourceName)) + } + return results +} + +func annotatePod(ctx context.Context, c k8s.Client, pod corev1.Pod, expectedLabels []string, resourceName string) error { + scheduled, nodeName := isPodScheduled(&pod) + if !scheduled { + return nil + } + node := &corev1.Node{} + if err := c.Get(ctx, types.NamespacedName{Name: nodeName}, node); err != nil { + return err + } + podAnnotations, err := getPodAnnotations(&pod, expectedLabels, node.Labels) + if err != nil { + return err + } + if len(podAnnotations) == 0 { + return nil + } + ulog.FromContext(ctx).Info( + "Setting Pod annotations from node labels", + "namespace", pod.Namespace, + "resource_name", resourceName, + "pod", pod.Name, + "annotations", podAnnotations, + ) + mergePatch, err := json.Marshal(map[string]any{ + "metadata": map[string]any{ + "annotations": podAnnotations, + }, + }) + if err != nil { + return err + } + if err := c.Patch(ctx, &pod, client.RawPatch(types.StrategicMergePatchType, mergePatch)); err != nil && !errors.IsNotFound(err) { + return err + } + return nil +} + +// isPodScheduled returns whether the Pod is scheduled and the node it is scheduled on. +func isPodScheduled(pod *corev1.Pod) (bool, string) { + for _, cond := range pod.Status.Conditions { + if cond.Type != corev1.PodScheduled { + continue + } + return cond.Status == corev1.ConditionTrue && pod.Spec.NodeName != "", pod.Spec.NodeName + } + return false, "" +} + +// getPodAnnotations returns missing annotations to add on the given Pod derived from its node labels. +// Labels that are expected but missing from the node result in an error. +func getPodAnnotations(pod *corev1.Pod, expectedAnnotations []string, nodeLabels map[string]string) (map[string]string, error) { + podAnnotations := make(map[string]string) + var missingLabels []string + for _, expectedAnnotation := range expectedAnnotations { + value, ok := nodeLabels[expectedAnnotation] + if !ok { + missingLabels = append(missingLabels, expectedAnnotation) + continue + } + if _, alreadyExists := pod.Annotations[expectedAnnotation]; alreadyExists { + continue + } + podAnnotations[expectedAnnotation] = value + } + if len(missingLabels) > 0 { + return nil, fmt.Errorf( + "following annotations are expected to be set on Pod %s/%s but do not exist as node labels: %s", + pod.Namespace, + pod.Name, + strings.Join(missingLabels, ","), + ) + } + return podAnnotations, nil +} diff --git a/pkg/controller/common/nodelabels/nodelabels_test.go b/pkg/controller/common/nodelabels/nodelabels_test.go new file mode 100644 index 00000000000..0d474d8ddb3 --- /dev/null +++ b/pkg/controller/common/nodelabels/nodelabels_test.go @@ -0,0 +1,131 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package nodelabels + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s" +) + +func TestAnnotatePods(t *testing.T) { + const namespace = "ns" + podSelector := map[string]string{"app": "sample"} + + newPod := func(name, nodeName string, annotations map[string]string) *corev1.Pod { + return &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + Labels: podSelector, + Annotations: annotations, + }, + Spec: corev1.PodSpec{NodeName: nodeName}, + Status: corev1.PodStatus{ + Conditions: []corev1.PodCondition{ + {Type: corev1.PodScheduled, Status: corev1.ConditionTrue}, + }, + }, + } + } + + node0 := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "k8s-node-0", + Labels: map[string]string{"topology.kubernetes.io/region": "europe-west1", "topology.kubernetes.io/zone": "europe-west1-a"}, + }} + node1 := &corev1.Node{ObjectMeta: metav1.ObjectMeta{ + Name: "k8s-node-1", + Labels: map[string]string{"topology.kubernetes.io/region": "europe-west1", "topology.kubernetes.io/zone": "europe-west1-b"}, + }} + + tests := []struct { + name string + expectedLabels []string + objects []client.Object + wantErrMsg string + wantAnnots map[string]map[string]string + }{ + { + name: "no expected labels noop", + expectedLabels: nil, + objects: []client.Object{ + newPod("p0", "k8s-node-0", nil), + node0, + }, + wantAnnots: map[string]map[string]string{"p0": nil}, + }, + { + name: "node missing labels returns error", + expectedLabels: []string{"topology.kubernetes.io/region"}, + objects: []client.Object{ + newPod("p0", "k8s-node-0", nil), + &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "k8s-node-0"}}, + }, + wantErrMsg: "following annotations are expected to be set on Pod ns/p0 but do not exist as node labels: topology.kubernetes.io/region", + }, + { + name: "annotates pods with node labels", + expectedLabels: []string{"topology.kubernetes.io/region", "topology.kubernetes.io/zone"}, + objects: []client.Object{ + newPod("p0", "k8s-node-0", map[string]string{"existing": "value"}), + newPod("p1", "k8s-node-1", nil), + node0, + node1, + }, + wantAnnots: map[string]map[string]string{ + "p0": { + "existing": "value", + "topology.kubernetes.io/region": "europe-west1", + "topology.kubernetes.io/zone": "europe-west1-a", + }, + "p1": { + "topology.kubernetes.io/region": "europe-west1", + "topology.kubernetes.io/zone": "europe-west1-b", + }, + }, + }, + { + name: "retains existing pod annotation value", + expectedLabels: []string{"topology.kubernetes.io/region"}, + objects: []client.Object{ + newPod("p0", "k8s-node-0", map[string]string{"topology.kubernetes.io/region": "manual-value"}), + node0, + }, + wantAnnots: map[string]map[string]string{ + "p0": {"topology.kubernetes.io/region": "manual-value"}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := k8s.NewFakeClient(tt.objects...) + results := AnnotatePods(context.Background(), c, namespace, podSelector, tt.expectedLabels, "sample") + _, err := results.Aggregate() + if tt.wantErrMsg != "" { + assert.ErrorContains(t, err, tt.wantErrMsg) + return + } + assert.NoError(t, err) + pods := &corev1.PodList{} + assert.NoError(t, c.List(context.Background(), pods)) + for _, pod := range pods.Items { + wantAnnots, ok := tt.wantAnnots[pod.Name] + if !ok { + continue + } + for k, v := range wantAnnots { + assert.Equal(t, v, pod.Annotations[k], "pod %s annotation %s", pod.Name, k) + } + } + }) + } +} diff --git a/pkg/controller/kibana/driver.go b/pkg/controller/kibana/driver.go index 0a0adb55c45..56542508ffc 100644 --- a/pkg/controller/kibana/driver.go +++ b/pkg/controller/kibana/driver.go @@ -28,6 +28,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/events" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" @@ -212,6 +213,17 @@ func (d *driver) Reconcile( } state.Kibana.Status.DeploymentStatus = deploymentStatus + // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the + // reconciliation loop as we don't want to prevent other updates from being applied. + results.WithResults(nodelabels.AnnotatePods( + ctx, + d.K8sClient(), + kb.Namespace, + map[string]string{kblabel.KibanaNameLabelName: kb.Name}, + kb.DownwardNodeLabels(), + kb.Name, + )) + return results } @@ -278,6 +290,12 @@ func (d *driver) deploymentParams(ctx context.Context, kb *kbv1.Kibana, policyAn return deployment.Params{}, err } + // Changes to the downward-node-labels annotation must roll the Kibana Pods so the new annotations + // are re-applied on scheduling. + if kb.HasDownwardNodeLabels() { + _, _ = configHash.Write([]byte(kb.Annotations[kbv1.DownwardNodeLabelsAnnotation])) + } + if kb.Spec.HTTP.TLS.Enabled() { // fetch the secret to calculate the checksum var httpCerts corev1.Secret diff --git a/pkg/controller/logstash/driver.go b/pkg/controller/logstash/driver.go index e4d9abbf26e..30bb757120d 100644 --- a/pkg/controller/logstash/driver.go +++ b/pkg/controller/logstash/driver.go @@ -21,6 +21,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/expectations" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/statefulset" @@ -154,7 +155,20 @@ func internalReconcile(params Params) (*reconciler.Results, logstashv1alpha1.Log if err != nil { return results.WithError(err), params.Status } - return reconcileStatefulSet(params, podTemplate) + statefulSetResults, status := reconcileStatefulSet(params, podTemplate) + + // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the + // reconciliation loop as we don't want to prevent other updates from being applied. + statefulSetResults.WithResults(nodelabels.AnnotatePods( + params.Context, + params.Client, + params.Logstash.Namespace, + map[string]string{labels.NameLabelName: params.Logstash.Name}, + params.Logstash.DownwardNodeLabels(), + params.Logstash.Name, + )) + + return statefulSetResults, status } // expectationsSatisfied checks that resources in our local cache match what we expect. diff --git a/pkg/controller/logstash/pod.go b/pkg/controller/logstash/pod.go index 4a203029eb0..37b6d62da14 100644 --- a/pkg/controller/logstash/pod.go +++ b/pkg/controller/logstash/pod.go @@ -89,6 +89,12 @@ func buildPodTemplate(params Params, configHash hash.Hash32) (corev1.PodTemplate return corev1.PodTemplateSpec{}, err } + // Changes to the downward-node-labels annotation must roll the Logstash Pods so the new annotations + // are re-applied on scheduling. + if params.Logstash.HasDownwardNodeLabels() { + _, _ = configHash.Write([]byte(params.Logstash.Annotations[logstashv1alpha1.DownwardNodeLabelsAnnotation])) + } + annotations := map[string]string{ ConfigHashAnnotationName: fmt.Sprint(configHash.Sum32()), } diff --git a/pkg/utils/nodelabels/nodelabels.go b/pkg/utils/nodelabels/nodelabels.go new file mode 100644 index 00000000000..daf7f947cce --- /dev/null +++ b/pkg/utils/nodelabels/nodelabels.go @@ -0,0 +1,40 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +// Package nodelabels provides the constant and parsing helpers used by the ECK resources to +// declare the set of Kubernetes node labels that must be copied to the annotations of the Pods +// managed by a given resource. +package nodelabels + +import ( + "strings" + + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/set" +) + +// DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels to +// be set as annotations on the Pods managed by an ECK resource. +const DownwardNodeLabelsAnnotation = "eck.k8s.elastic.co/downward-node-labels" + +// Parse normalizes a comma-separated node labels annotation value into a sorted, deduplicated +// slice. An empty or whitespace-only value returns nil. +func Parse(annotationValue string) []string { + labels := set.Make() + for label := range strings.SplitSeq(annotationValue, ",") { + label = strings.TrimSpace(label) + if label == "" { + continue + } + labels.Add(label) + } + if labels.Count() == 0 { + return nil + } + return labels.AsSortedSlice() +} + +// FromAnnotations returns the list of downward node labels declared via DownwardNodeLabelsAnnotation. +func FromAnnotations(annotations map[string]string) []string { + return Parse(annotations[DownwardNodeLabelsAnnotation]) +} diff --git a/pkg/utils/nodelabels/nodelabels_test.go b/pkg/utils/nodelabels/nodelabels_test.go new file mode 100644 index 00000000000..1250a3ee930 --- /dev/null +++ b/pkg/utils/nodelabels/nodelabels_test.go @@ -0,0 +1,42 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package nodelabels + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestParse(t *testing.T) { + tests := []struct { + name, input string + want []string + }{ + {name: "empty", input: "", want: nil}, + {name: "whitespace only", input: " , , ", want: nil}, + {name: "single", input: "topology.kubernetes.io/zone", want: []string{"topology.kubernetes.io/zone"}}, + { + name: "trimmed deduplicated and sorted", + input: " zeta.io/rack ,alpha.io/zone,alpha.io/zone, ", + want: []string{"alpha.io/zone", "zeta.io/rack"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + assert.Equal(t, tt.want, Parse(tt.input)) + }) + } +} + +func TestFromAnnotations(t *testing.T) { + assert.Nil(t, FromAnnotations(nil)) + assert.Nil(t, FromAnnotations(map[string]string{"other": "value"})) + assert.Equal( + t, + []string{"topology.kubernetes.io/region", "topology.kubernetes.io/zone"}, + FromAnnotations(map[string]string{DownwardNodeLabelsAnnotation: "topology.kubernetes.io/zone,topology.kubernetes.io/region"}), + ) +} From 7bb9a37304c92513d924523050311bb3bcff14a5 Mon Sep 17 00:00:00 2001 From: Joel Teo Date: Mon, 20 Apr 2026 23:00:32 +0800 Subject: [PATCH 2/4] Gate Kibana/Logstash/Agent start on node-label annotations being set Address PR feedback: Elasticsearch's prepare-fs init container waits until the operator has patched the expected node-derived keys onto the Pod's metadata.annotations (it polls the downward-API annotations file) before the main ES container starts. The previous change for Kibana, Logstash and Elastic Agent added the same operator patch behavior but not that startup gate, so anything that consumes those annotations at first container start (for example valueFrom.fieldRef on metadata.annotations[...]) could still race on first start. This commit adds a shared wait-for-annotations init container that polls the downward-API annotations file and only lets the main container start once all expected annotations are present. The init container is added to the Pod template for Kibana, Logstash and Elastic Agent when downward node labels are configured on the CR. --- pkg/controller/agent/pod.go | 16 ++++ .../common/nodelabels/initcontainer.go | 74 +++++++++++++++++++ .../common/nodelabels/initcontainer_test.go | 46 ++++++++++++ pkg/controller/kibana/pod.go | 15 ++++ pkg/controller/logstash/pod.go | 19 ++++- 5 files changed, 169 insertions(+), 1 deletion(-) create mode 100644 pkg/controller/common/nodelabels/initcontainer.go create mode 100644 pkg/controller/common/nodelabels/initcontainer_test.go diff --git a/pkg/controller/agent/pod.go b/pkg/controller/agent/pod.go index 279b6dc2734..6b323842e0c 100644 --- a/pkg/controller/agent/pod.go +++ b/pkg/controller/agent/pod.go @@ -30,6 +30,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/defaults" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/labels" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" @@ -202,6 +203,21 @@ func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret }}, ) + // Block Pod start on the operator patching the expected node-label annotations onto the + // Pod, so the Agent container does not start while those annotations are missing from + // the downward-API annotations file. + if params.Agent.HasDownwardNodeLabels() { + downwardAPIVolume := commonnodelabels.DownwardAPIVolume() + waitInit, err := commonnodelabels.WaitForAnnotationsInitContainer(params.Agent.DownwardNodeLabels()) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + builder = builder. + WithVolumes(downwardAPIVolume.Volume()). + WithInitContainers(waitInit). + WithInitContainerDefaults() + } + return builder.PodTemplate, nil } diff --git a/pkg/controller/common/nodelabels/initcontainer.go b/pkg/controller/common/nodelabels/initcontainer.go new file mode 100644 index 00000000000..c180a8c72a6 --- /dev/null +++ b/pkg/controller/common/nodelabels/initcontainer.go @@ -0,0 +1,74 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package nodelabels + +import ( + "bytes" + "fmt" + "strings" + "text/template" + + corev1 "k8s.io/api/core/v1" + + commonvolume "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/volume" + esvolume "github.com/elastic/cloud-on-k8s/v3/pkg/controller/elasticsearch/volume" +) + +// WaitForAnnotationsContainerName is the name of the init container that blocks Pod start +// until the operator has patched the expected node-derived annotations onto the Pod. +const WaitForAnnotationsContainerName = "elastic-internal-wait-for-node-labels" + +var waitScriptTemplate = template.Must(template.New("").Parse( + `#!/usr/bin/env bash +set -eu +expected_annotations=({{ .ExpectedAnnotations }}) +annotations_file={{ .AnnotationsFile }} +function annotations_exist() { + for expected_annotation in "${expected_annotations[@]}"; do + if ! grep -qE "^${expected_annotation}=" "${annotations_file}"; then + return 1 + fi + done + return 0 +} +echo "Waiting for the following annotations to be set on Pod: {{ .ExpectedAnnotations }}" +while ! annotations_exist; do sleep 2; done +echo "All expected annotations are set." +`)) + +// DownwardAPIVolume returns the downward API volume that exposes the Pod annotations file +// under the path polled by the wait-for-annotations init container. +func DownwardAPIVolume() commonvolume.DownwardAPI { + return commonvolume.DownwardAPI{}.WithAnnotations(true) +} + +// WaitForAnnotationsInitContainer builds an init container that blocks until the operator +// has patched all expectedAnnotations onto the Pod's metadata.annotations. This mirrors the +// behavior of the Elasticsearch prepare-fs init container and prevents the main container +// from starting while the downward-API annotations file is still missing labels that users +// may consume via `valueFrom.fieldRef`. +// +// The image and resources are left unset so they are inherited from the main container via +// PodTemplateBuilder.WithInitContainerDefaults. Callers must also add the volume returned +// by DownwardAPIVolume to the Pod. +func WaitForAnnotationsInitContainer(expectedAnnotations []string) (corev1.Container, error) { + buf := bytes.Buffer{} + if err := waitScriptTemplate.Execute(&buf, struct { + ExpectedAnnotations string + AnnotationsFile string + }{ + ExpectedAnnotations: strings.Join(expectedAnnotations, " "), + AnnotationsFile: fmt.Sprintf("%s/%s", esvolume.DownwardAPIMountPath, esvolume.AnnotationsFile), + }); err != nil { + return corev1.Container{}, err + } + return corev1.Container{ + Name: WaitForAnnotationsContainerName, + Command: []string{"bash", "-c", buf.String()}, + VolumeMounts: []corev1.VolumeMount{ + DownwardAPIVolume().VolumeMount(), + }, + }, nil +} diff --git a/pkg/controller/common/nodelabels/initcontainer_test.go b/pkg/controller/common/nodelabels/initcontainer_test.go new file mode 100644 index 00000000000..8d90a59e803 --- /dev/null +++ b/pkg/controller/common/nodelabels/initcontainer_test.go @@ -0,0 +1,46 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package nodelabels + +import ( + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestWaitForAnnotationsInitContainer(t *testing.T) { + c, err := WaitForAnnotationsInitContainer([]string{"topology.kubernetes.io/zone", "topology.kubernetes.io/region"}) + require.NoError(t, err) + assert.Equal(t, WaitForAnnotationsContainerName, c.Name) + require.Len(t, c.VolumeMounts, 1) + assert.Equal(t, "downward-api", c.VolumeMounts[0].Name) + assert.Equal(t, "/mnt/elastic-internal/downward-api", c.VolumeMounts[0].MountPath) + assert.True(t, c.VolumeMounts[0].ReadOnly) + + require.Len(t, c.Command, 3) + assert.Equal(t, "bash", c.Command[0]) + assert.Equal(t, "-c", c.Command[1]) + script := c.Command[2] + assert.Contains(t, script, "topology.kubernetes.io/zone topology.kubernetes.io/region") + assert.Contains(t, script, "/mnt/elastic-internal/downward-api/annotations") + // Each expected annotation is matched at the beginning of a line followed by '=': + assert.Contains(t, script, `grep -qE "^${expected_annotation}="`) + // No image/resources are set so they are inherited from the main container. + assert.Equal(t, "", c.Image) + assert.Empty(t, c.Resources.Limits) + assert.Empty(t, c.Resources.Requests) +} + +func TestDownwardAPIVolume_IncludesAnnotations(t *testing.T) { + v := DownwardAPIVolume().Volume() + require.NotNil(t, v.VolumeSource.DownwardAPI) + paths := make([]string, 0, len(v.VolumeSource.DownwardAPI.Items)) + for _, item := range v.VolumeSource.DownwardAPI.Items { + paths = append(paths, item.Path) + } + assert.Contains(t, strings.Join(paths, ","), "annotations") +} diff --git a/pkg/controller/kibana/pod.go b/pkg/controller/kibana/pod.go index 80cfd0e0992..5b23a122773 100644 --- a/pkg/controller/kibana/pod.go +++ b/pkg/controller/kibana/pod.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/defaults" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/pod" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/settings" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" @@ -157,6 +158,20 @@ func NewPodTemplateSpec( builder.WithInitContainers(initContainer) + // Block Pod start on the operator patching the expected node-label annotations onto the + // Pod, so the Kibana container does not start while those annotations are missing from + // the downward-API annotations file. + if kb.HasDownwardNodeLabels() { + downwardAPIVolume := commonnodelabels.DownwardAPIVolume() + waitInit, err := commonnodelabels.WaitForAnnotationsInitContainer(kb.DownwardNodeLabels()) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + builder. + WithVolumes(downwardAPIVolume.Volume()). + WithInitContainers(waitInit) + } + // Kibana 7.5.0 and above support running with a read-only root filesystem, // but require a temporary volume to be mounted at /tmp for some reporting features // and a plugin volume mounted at /usr/share/kibana/plugins. Also needed is an diff --git a/pkg/controller/logstash/pod.go b/pkg/controller/logstash/pod.go index 37b6d62da14..df531846923 100644 --- a/pkg/controller/logstash/pod.go +++ b/pkg/controller/logstash/pod.go @@ -23,6 +23,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/defaults" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/logstash/network" @@ -128,7 +129,23 @@ func buildPodTemplate(params Params, configHash hash.Hash32) (corev1.PodTemplate WithEnv(envs...). WithVolumes(volumes...). WithVolumeMounts(volumeMounts...). - WithInitContainers(initConfigContainer(params)). + WithInitContainers(initConfigContainer(params)) + + // Block Pod start on the operator patching the expected node-label annotations onto the + // Pod, so the Logstash container does not start while those annotations are missing from + // the downward-API annotations file. + if params.Logstash.HasDownwardNodeLabels() { + downwardAPIVolume := commonnodelabels.DownwardAPIVolume() + waitInit, err := commonnodelabels.WaitForAnnotationsInitContainer(params.Logstash.DownwardNodeLabels()) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + builder = builder. + WithVolumes(downwardAPIVolume.Volume()). + WithInitContainers(waitInit) + } + + builder = builder. WithInitContainerDefaults(). WithPodSecurityContext(DefaultSecurityContext) From f496ddce311bd2d751304befbf2dc3f14d28d6a3 Mon Sep 17 00:00:00 2001 From: Joel Teo Date: Wed, 27 May 2026 13:03:06 +0800 Subject: [PATCH 3/4] Address review feedback on downward-node-labels propagation - Extract an AnnotationTarget interface in pkg/controller/common/nodelabels so the driver call collapses to AnnotatePods(ctx, client, &resource), with each CR satisfying the interface via metav1.Object, GetIdentityLabels and DownwardNodeLabels. - Reuse the Elasticsearch annotation-policy validation across CRDs. Move NodeLabels and NewExposedNodeLabels into the shared nodelabels package and add ValidateAnnotation. Wire a WithNodeLabelsValidation webhook wrapper so Kibana/Agent/APM/Beat/Maps/Package Registry validators enforce the exposed-node-labels policy, and add the same check to each controller's validate() to cover the webhook-disabled path. Update docs/reference/eck-configuration-flags.md. - Extend support to Beat, APM Server, Elastic Maps Server and Elastic Package Registry: add DownwardNodeLabels/HasDownwardNodeLabels accessors, propagate annotations to managed Pods, include the WaitFor init container and downward-API volume on the Pod template, and roll Pods on annotation changes via the config hash. --- cmd/manager/validation.go | 21 ++++-- docs/reference/eck-configuration-flags.md | 2 +- pkg/apis/apm/v1/apmserver_types.go | 15 +++++ pkg/apis/beat/v1beta1/beat_types.go | 15 +++++ pkg/apis/maps/v1alpha1/maps_types.go | 15 +++++ .../packageregistry/v1alpha1/epr_types.go | 15 +++++ pkg/controller/agent/controller.go | 8 +++ pkg/controller/agent/driver.go | 9 +-- pkg/controller/apmserver/controller.go | 12 ++++ pkg/controller/apmserver/deployment.go | 6 ++ pkg/controller/apmserver/pod.go | 15 +++++ pkg/controller/beat/common/driver.go | 6 ++ pkg/controller/beat/common/pod.go | 14 ++++ pkg/controller/beat/controller.go | 8 +++ pkg/controller/common/nodelabels/exposed.go | 66 +++++++++++++++++++ .../common/nodelabels/nodelabels.go | 34 ++++++---- .../common/nodelabels/nodelabels_test.go | 17 ++++- .../common/webhook/resource_validator.go | 30 +++++++++ .../elasticsearch/validation/node_labels.go | 30 ++------- .../elasticsearch/validation/validations.go | 21 +----- pkg/controller/kibana/controller.go | 8 +++ pkg/controller/kibana/driver.go | 9 +-- pkg/controller/logstash/driver.go | 9 +-- .../logstash/logstash_controller.go | 2 +- pkg/controller/logstash/validation/webhook.go | 12 ++-- pkg/controller/maps/controller.go | 18 +++++ pkg/controller/maps/pod.go | 15 +++++ pkg/controller/packageregistry/controller.go | 18 +++++ pkg/controller/packageregistry/pod.go | 15 +++++ 29 files changed, 371 insertions(+), 94 deletions(-) create mode 100644 pkg/controller/common/nodelabels/exposed.go diff --git a/cmd/manager/validation.go b/cmd/manager/validation.go index 5dc5627b5fb..1069e4cfbc8 100644 --- a/cmd/manager/validation.go +++ b/cmd/manager/validation.go @@ -15,6 +15,7 @@ import ( "github.com/spf13/viper" "go.elastic.co/apm/v2" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/manager" @@ -87,17 +88,23 @@ func setupWebhook( checker := commonlicense.NewLicenseChecker(mgr.GetClient(), params.OperatorNamespace) // setup webhooks for supported types - commonwebhook.RegisterResourceWebhook(mgr, agentv1alpha1.WebhookPath, checker, managedNamespaces, agentv1alpha1.Validate, "Agent") - commonwebhook.RegisterResourceWebhook(mgr, apmv1.WebhookPath, checker, managedNamespaces, apmv1.Validate, "APM Server") + commonwebhook.RegisterResourceWebhook(mgr, agentv1alpha1.WebhookPath, checker, managedNamespaces, + commonwebhook.WithNodeLabelsValidation(agentv1alpha1.Validate, exposedNodeLabels, schema.GroupKind{Group: agentv1alpha1.GroupVersion.Group, Kind: agentv1alpha1.Kind}), "Agent") + commonwebhook.RegisterResourceWebhook(mgr, apmv1.WebhookPath, checker, managedNamespaces, + commonwebhook.WithNodeLabelsValidation(apmv1.Validate, exposedNodeLabels, schema.GroupKind{Group: apmv1.GroupVersion.Group, Kind: apmv1.Kind}), "APM Server") commonwebhook.RegisterResourceWebhook(mgr, apmv1beta1.WebhookPath, checker, managedNamespaces, apmv1beta1.Validate, "APM Server") - commonwebhook.RegisterResourceWebhook(mgr, beatv1beta1.WebhookPath, checker, managedNamespaces, beatv1beta1.Validate, "Beat") + commonwebhook.RegisterResourceWebhook(mgr, beatv1beta1.WebhookPath, checker, managedNamespaces, + commonwebhook.WithNodeLabelsValidation(beatv1beta1.Validate, exposedNodeLabels, schema.GroupKind{Group: beatv1beta1.GroupVersion.Group, Kind: beatv1beta1.Kind}), "Beat") commonwebhook.RegisterResourceWebhook(mgr, entv1.WebhookPath, checker, managedNamespaces, entv1.Validate, "Enterprise Search") commonwebhook.RegisterResourceWebhook(mgr, entv1beta1.WebhookPath, checker, managedNamespaces, entv1beta1.Validate, "Enterprise Search") commonwebhook.RegisterResourceWebhook(mgr, esv1beta1.WebhookPath, checker, managedNamespaces, esv1beta1.Validate, "Elasticsearch") - commonwebhook.RegisterResourceWebhook(mgr, kbv1.WebhookPath, checker, managedNamespaces, kbv1.Validate, "Kibana") + commonwebhook.RegisterResourceWebhook(mgr, kbv1.WebhookPath, checker, managedNamespaces, + commonwebhook.WithNodeLabelsValidation(kbv1.Validate, exposedNodeLabels, schema.GroupKind{Group: kbv1.GroupVersion.Group, Kind: kbv1.Kind}), "Kibana") commonwebhook.RegisterResourceWebhook(mgr, kbv1beta1.WebhookPath, checker, managedNamespaces, kbv1beta1.Validate, "Kibana") - commonwebhook.RegisterResourceWebhook(mgr, emsv1alpha1.WebhookPath, checker, managedNamespaces, emsv1alpha1.Validate, "Elastic Maps Server") - commonwebhook.RegisterResourceWebhook(mgr, eprv1alpha1.WebhookPath, checker, managedNamespaces, eprv1alpha1.Validate, "Package Registry") + commonwebhook.RegisterResourceWebhook(mgr, emsv1alpha1.WebhookPath, checker, managedNamespaces, + commonwebhook.WithNodeLabelsValidation(emsv1alpha1.Validate, exposedNodeLabels, schema.GroupKind{Group: emsv1alpha1.GroupVersion.Group, Kind: emsv1alpha1.Kind}), "Elastic Maps Server") + commonwebhook.RegisterResourceWebhook(mgr, eprv1alpha1.WebhookPath, checker, managedNamespaces, + commonwebhook.WithNodeLabelsValidation(eprv1alpha1.Validate, exposedNodeLabels, schema.GroupKind{Group: eprv1alpha1.GroupVersion.Group, Kind: eprv1alpha1.Kind}), "Package Registry") commonwebhook.RegisterResourceWebhook(mgr, policyv1alpha1.WebhookPath, checker, managedNamespaces, policyv1alpha1.Validate, "Stack Config Policy") // Logstash, Elasticsearch v1, ElasticsearchAutoscaling, and AutoOps validating webhooks are wired up @@ -105,7 +112,7 @@ func setupWebhook( // v1beta1 remains in the RegisterResourceWebhook list above because it only needs a ValidateFunc. esvalidation.RegisterWebhook(mgr, params.ValidateStorageClass, exposedNodeLabels, checker, managedNamespaces) esavalidation.RegisterWebhook(mgr, params.ValidateStorageClass, checker, managedNamespaces) - lsvalidation.RegisterWebhook(mgr, params.ValidateStorageClass, managedNamespaces) + lsvalidation.RegisterWebhook(mgr, params.ValidateStorageClass, exposedNodeLabels, managedNamespaces) autoopsvalidation.RegisterWebhook(mgr, checker, managedNamespaces) // wait for the secret to be populated in the local filesystem before returning diff --git a/docs/reference/eck-configuration-flags.md b/docs/reference/eck-configuration-flags.md index c16552c98a3..f96e373cb60 100644 --- a/docs/reference/eck-configuration-flags.md +++ b/docs/reference/eck-configuration-flags.md @@ -29,7 +29,7 @@ The following table lists and describes all the available configuration flags fo | `enable-tracing` | `false` | Enable APM tracing in the operator process. Use environment variables to configure APM server URL, credentials, and so on. Check [Apm Go Agent reference](apm-agent-go://reference/configuration.md) for details. | | `enable-webhook` | `false` | Enables a validating webhook server in the operator process. | | `enforce-rbac-on-refs` | `false` | Enables restrictions on cross-namespace resource association through RBAC. | -| `exposed-node-labels` | `""` | List of Kubernetes node labels which are allowed to be copied as annotations on the Elasticsearch Pods. Check [Topology spread constraints and availability zone awareness](docs-content://deploy-manage/deploy/cloud-on-k8s/advanced-elasticsearch-node-scheduling.md#k8s-availability-zone-awareness) for more details. | +| `exposed-node-labels` | `""` | List of Kubernetes node labels which are allowed to be copied as annotations on the Pods managed by ECK (Elasticsearch, Kibana, Logstash, Elastic Agent, Beat, APM Server, Elastic Maps Server, and Package Registry). Check [Topology spread constraints and availability zone awareness](docs-content://deploy-manage/deploy/cloud-on-k8s/advanced-elasticsearch-node-scheduling.md#k8s-availability-zone-awareness) for more details. | | `ip-family` | `""` | Set the IP family to use. Possible values: IPv4, IPv6, "" (= auto-detect) | | `kube-client-qps` | `0` | Set the maximum number of queries per second to the Kubernetes API. Default value is inherited from the [Go client](https://github.com/kubernetes/client-go/blob/e6538dd42b4fe55b6c754e41c66b43133ba41a59/rest/config.go#L44). | | `kube-client-timeout` | `60s` | Set the request timeout for Kubernetes API calls made by the operator. | diff --git a/pkg/apis/apm/v1/apmserver_types.go b/pkg/apis/apm/v1/apmserver_types.go index 1143a57a8f3..5bae807a707 100644 --- a/pkg/apis/apm/v1/apmserver_types.go +++ b/pkg/apis/apm/v1/apmserver_types.go @@ -12,6 +12,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -19,6 +20,9 @@ const ( // Kind is inferred from the struct name using reflection in SchemeBuilder.Register() // we duplicate it as a constant here for practical purposes. Kind = "ApmServer" + // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels + // to be set as annotations on the APM Server Pods. + DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // ApmServerSpec holds the specification of an APM Server. @@ -132,6 +136,17 @@ func (as *ApmServer) IsMarkedForDeletion() bool { return !as.DeletionTimestamp.IsZero() } +// DownwardNodeLabels returns the node labels to copy as annotations on the APM Server Pods, +// as declared via the DownwardNodeLabelsAnnotation annotation. +func (as *ApmServer) DownwardNodeLabels() []string { + return nodelabels.FromAnnotations(as.Annotations) +} + +// HasDownwardNodeLabels returns true if node labels are expected to be propagated to the APM Server Pods. +func (as *ApmServer) HasDownwardNodeLabels() bool { + return len(as.DownwardNodeLabels()) > 0 +} + func (as *ApmServer) SecureSettings() []commonv1.SecretSource { return as.Spec.SecureSettings } diff --git a/pkg/apis/beat/v1beta1/beat_types.go b/pkg/apis/beat/v1beta1/beat_types.go index e95abfe951f..52d1c459774 100644 --- a/pkg/apis/beat/v1beta1/beat_types.go +++ b/pkg/apis/beat/v1beta1/beat_types.go @@ -13,12 +13,16 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/stackmon/monitoring" + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( // Kind is inferred from the struct name using reflection in SchemeBuilder.Register() // we duplicate it as a constant here for practical purposes. Kind = "Beat" + // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels + // to be set as annotations on the Beat Pods. + DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) var ( @@ -278,6 +282,17 @@ func (b *Beat) IsMarkedForDeletion() bool { return !b.DeletionTimestamp.IsZero() } +// DownwardNodeLabels returns the node labels to copy as annotations on the Beat Pods, +// as declared via the DownwardNodeLabelsAnnotation annotation. +func (b *Beat) DownwardNodeLabels() []string { + return nodelabels.FromAnnotations(b.Annotations) +} + +// HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Beat Pods. +func (b *Beat) HasDownwardNodeLabels() bool { + return len(b.DownwardNodeLabels()) > 0 +} + func (b *Beat) ElasticsearchRef() commonv1.ElasticsearchSelector { return b.Spec.ElasticsearchRef } diff --git a/pkg/apis/maps/v1alpha1/maps_types.go b/pkg/apis/maps/v1alpha1/maps_types.go index 2a69ea41925..57b0ed5ab06 100644 --- a/pkg/apis/maps/v1alpha1/maps_types.go +++ b/pkg/apis/maps/v1alpha1/maps_types.go @@ -11,6 +11,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -18,6 +19,9 @@ const ( // Kind is inferred from the struct name using reflection in SchemeBuilder.Register() // we duplicate it as a constant here for practical purposes. Kind = "ElasticMapsServer" + // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels + // to be set as annotations on the Elastic Maps Server Pods. + DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // MapsSpec holds the specification of an Elastic Maps Server instance. @@ -84,6 +88,17 @@ func (m *ElasticMapsServer) IsMarkedForDeletion() bool { return !m.DeletionTimestamp.IsZero() } +// DownwardNodeLabels returns the node labels to copy as annotations on the Elastic Maps Server Pods, +// as declared via the DownwardNodeLabelsAnnotation annotation. +func (m *ElasticMapsServer) DownwardNodeLabels() []string { + return nodelabels.FromAnnotations(m.Annotations) +} + +// HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Elastic Maps Server Pods. +func (m *ElasticMapsServer) HasDownwardNodeLabels() bool { + return len(m.DownwardNodeLabels()) > 0 +} + func (m *ElasticMapsServer) Associated() commonv1.Associated { return m } diff --git a/pkg/apis/packageregistry/v1alpha1/epr_types.go b/pkg/apis/packageregistry/v1alpha1/epr_types.go index b318ff1a696..04cfa468bd4 100644 --- a/pkg/apis/packageregistry/v1alpha1/epr_types.go +++ b/pkg/apis/packageregistry/v1alpha1/epr_types.go @@ -10,6 +10,7 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" common_name "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/name" + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -17,6 +18,9 @@ const ( // Kind is inferred from the struct name using reflection in SchemeBuilder.Register() // we duplicate it as a constant here for practical purposes. Kind = "PackageRegistry" + // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels + // to be set as annotations on the Elastic Package Registry Pods. + DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // Namer is a Namer that is configured with the defaults for resources related to a Package Registry resource. @@ -95,6 +99,17 @@ func (m *PackageRegistry) IsMarkedForDeletion() bool { return !m.DeletionTimestamp.IsZero() } +// DownwardNodeLabels returns the node labels to copy as annotations on the Elastic Package Registry Pods, +// as declared via the DownwardNodeLabelsAnnotation annotation. +func (m *PackageRegistry) DownwardNodeLabels() []string { + return nodelabels.FromAnnotations(m.Annotations) +} + +// HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Elastic Package Registry Pods. +func (m *PackageRegistry) HasDownwardNodeLabels() bool { + return len(m.DownwardNodeLabels()) > 0 +} + // GetObservedGeneration will return the observedGeneration from the Elastic Package Registry status. func (m *PackageRegistry) GetObservedGeneration() int64 { return m.Status.ObservedGeneration diff --git a/pkg/controller/agent/controller.go b/pkg/controller/agent/controller.go index 07b1b914713..5dc266dcce1 100644 --- a/pkg/controller/agent/controller.go +++ b/pkg/controller/agent/controller.go @@ -12,6 +12,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -25,6 +26,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/events" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" @@ -227,6 +229,12 @@ func (r *ReconcileAgent) validate(ctx context.Context, agent agentv1alpha1.Agent k8s.MaybeEmitErrorEvent(r.recorder, err, &agent, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(ctx, err) } + if errs := commonnodelabels.ValidateAnnotation(agent.Annotations, r.ExposedNodeLabels); len(errs) > 0 { + err := apierrors.NewInvalid(schema.GroupKind{Group: agentv1alpha1.GroupVersion.Group, Kind: agentv1alpha1.Kind}, agent.Name, errs) + logconf.FromContext(ctx).Error(err, "Validation failed") + k8s.MaybeEmitErrorEvent(r.recorder, err, &agent, events.EventReasonValidation, events.EventActionValidation, err.Error()) + return tracing.CaptureError(ctx, err) + } for _, warning := range warnings { k8s.EmitEvent(r.recorder, &agent, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } diff --git a/pkg/controller/agent/driver.go b/pkg/controller/agent/driver.go index 093d244d7da..cccb5f0eedb 100644 --- a/pkg/controller/agent/driver.go +++ b/pkg/controller/agent/driver.go @@ -179,14 +179,7 @@ func internalReconcile(params Params) (*reconciler.Results, agentv1alpha1.AgentS // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the // reconciliation loop as we don't want to prevent other updates from being applied. - results.WithResults(nodelabels.AnnotatePods( - params.Context, - params.Client, - params.Agent.Namespace, - map[string]string{NameLabelName: params.Agent.Name}, - params.Agent.DownwardNodeLabels(), - params.Agent.Name, - )) + results.WithResults(nodelabels.AnnotatePods(params.Context, params.Client, ¶ms.Agent)) return results, status } diff --git a/pkg/controller/apmserver/controller.go b/pkg/controller/apmserver/controller.go index f3eeec25d7a..da498299321 100644 --- a/pkg/controller/apmserver/controller.go +++ b/pkg/controller/apmserver/controller.go @@ -17,6 +17,7 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -36,6 +37,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/labels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" commonpassword "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/password" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" @@ -283,6 +285,10 @@ func (r *ReconcileApmServer) doReconcile(ctx context.Context, as *apmv1.ApmServe return results.WithError(tracing.CaptureError(ctx, err)), state } + // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the + // reconciliation loop as we don't want to prevent other updates from being applied. + results.WithResults(commonnodelabels.AnnotatePods(ctx, r.K8sClient(), as)) + state.UpdateApmServerExternalService(*svc) _, err = results.WithError(err).Aggregate() @@ -300,6 +306,12 @@ func (r *ReconcileApmServer) validate(ctx context.Context, as *apmv1.ApmServer) k8s.MaybeEmitErrorEvent(r.recorder, err, as, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } + if errs := commonnodelabels.ValidateAnnotation(as.Annotations, r.ExposedNodeLabels); len(errs) > 0 { + err := apierrors.NewInvalid(schema.GroupKind{Group: apmv1.GroupVersion.Group, Kind: apmv1.Kind}, as.Name, errs) + log.Error(err, "Validation failed") + k8s.MaybeEmitErrorEvent(r.recorder, err, as, events.EventReasonValidation, events.EventActionValidation, err.Error()) + return tracing.CaptureError(vctx, err) + } for _, warning := range warnings { k8s.EmitEvent(r.recorder, as, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } diff --git a/pkg/controller/apmserver/deployment.go b/pkg/controller/apmserver/deployment.go index 5e2a7f3aa23..c606a900297 100644 --- a/pkg/controller/apmserver/deployment.go +++ b/pkg/controller/apmserver/deployment.go @@ -116,6 +116,12 @@ func buildConfigHash(c k8s.Client, as *apmv1.ApmServer, params PodSpecParams) (s // - in the APMServer configuration file content _, _ = configHash.Write(params.ConfigSecret.Data[ApmCfgSecretKey]) + // Changes to the downward-node-labels annotation must roll the APM Server Pods so the new annotations + // are re-applied on scheduling. + if as.HasDownwardNodeLabels() { + _, _ = configHash.Write([]byte(as.Annotations[apmv1.DownwardNodeLabelsAnnotation])) + } + // - in the APMServer keystore if params.keystoreResources != nil { _, _ = configHash.Write([]byte(params.keystoreResources.Hash)) diff --git a/pkg/controller/apmserver/pod.go b/pkg/controller/apmserver/pod.go index 2d21b020d61..45f104b8d42 100644 --- a/pkg/controller/apmserver/pod.go +++ b/pkg/controller/apmserver/pod.go @@ -19,6 +19,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/defaults" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/volume" "github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s" @@ -161,6 +162,20 @@ func newPodSpec(c k8s.Client, as *apmv1.ApmServer, p PodSpecParams, meta metadat } builder = withHTTPCertsVolume(builder, *as) + // Block Pod start on the operator patching the expected node-label annotations onto the + // Pod, so the APM Server container does not start while those annotations are missing from + // the downward-API annotations file. + if as.HasDownwardNodeLabels() { + downwardAPIVolume := commonnodelabels.DownwardAPIVolume() + waitInit, err := commonnodelabels.WaitForAnnotationsInitContainer(as.DownwardNodeLabels()) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + builder = builder. + WithVolumes(downwardAPIVolume.Volume()). + WithInitContainers(waitInit) + } + if setDefaultSecurityContext { builder = builder.WithPodSecurityContext(corev1.PodSecurityContext{ SeccompProfile: &corev1.SeccompProfile{ diff --git a/pkg/controller/beat/common/driver.go b/pkg/controller/beat/common/driver.go index 4df6652d659..ac8d73e7ef4 100644 --- a/pkg/controller/beat/common/driver.go +++ b/pkg/controller/beat/common/driver.go @@ -19,6 +19,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/driver" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/settings" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" @@ -114,5 +115,10 @@ func Reconcile( var reconcileResults *reconciler.Results reconcileResults, params.Status = reconcilePodVehicle(podTemplate, params, meta) results.WithResults(reconcileResults) + + // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the + // reconciliation loop as we don't want to prevent other updates from being applied. + results.WithResults(commonnodelabels.AnnotatePods(params.Context, params.Client, ¶ms.Beat)) + return results, params.Status } diff --git a/pkg/controller/beat/common/pod.go b/pkg/controller/beat/common/pod.go index 4a425e4dcb9..97a70810c59 100644 --- a/pkg/controller/beat/common/pod.go +++ b/pkg/controller/beat/common/pod.go @@ -20,6 +20,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/defaults" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/stackmon/monitoring" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" @@ -152,6 +153,19 @@ func buildPodTemplate( initContainers = append(initContainers, keystoreResources.InitContainer) } + // Changes to the downward-node-labels annotation must roll the Beat Pods so the new annotations + // are re-applied on scheduling. + if params.Beat.HasDownwardNodeLabels() { + _, _ = configHash.Write([]byte(params.Beat.Annotations[beatv1beta1.DownwardNodeLabelsAnnotation])) + downwardAPIVolume := commonnodelabels.DownwardAPIVolume() + waitInit, err := commonnodelabels.WaitForAnnotationsInitContainer(params.Beat.DownwardNodeLabels()) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + volumes = append(volumes, downwardAPIVolume.Volume()) + initContainers = append(initContainers, waitInit) + } + if monitoring.IsLogsDefined(¶ms.Beat) { sideCar, err := beat_stackmon.Filebeat(params.Context, params.Client, ¶ms.Beat, params.Beat.Spec.Version, meta) if err != nil { diff --git a/pkg/controller/beat/controller.go b/pkg/controller/beat/controller.go index 9b57967f358..9be2ae40b39 100644 --- a/pkg/controller/beat/controller.go +++ b/pkg/controller/beat/controller.go @@ -11,6 +11,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -32,6 +33,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/events" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" @@ -196,6 +198,12 @@ func (r *ReconcileBeat) validate(ctx context.Context, beat *beatv1beta1.Beat) er k8s.MaybeEmitErrorEvent(r.recorder, err, beat, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } + if errs := commonnodelabels.ValidateAnnotation(beat.Annotations, r.ExposedNodeLabels); len(errs) > 0 { + err := apierrors.NewInvalid(schema.GroupKind{Group: beatv1beta1.GroupVersion.Group, Kind: beatv1beta1.Kind}, beat.Name, errs) + ulog.FromContext(ctx).Error(err, "Validation failed") + k8s.MaybeEmitErrorEvent(r.recorder, err, beat, events.EventReasonValidation, events.EventActionValidation, err.Error()) + return tracing.CaptureError(vctx, err) + } for _, warning := range warnings { k8s.EmitEvent(r.recorder, beat, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } diff --git a/pkg/controller/common/nodelabels/exposed.go b/pkg/controller/common/nodelabels/exposed.go new file mode 100644 index 00000000000..cac1fa40aea --- /dev/null +++ b/pkg/controller/common/nodelabels/exposed.go @@ -0,0 +1,66 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package nodelabels + +import ( + "fmt" + "regexp" + + "k8s.io/apimachinery/pkg/util/validation/field" + + "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" +) + +// NotAllowedNodesLabelMsg is returned when a node label requested via the downward-node-labels +// annotation is not allowed by the operator's exposed-node-labels policy. +const NotAllowedNodesLabelMsg = "Node label not in the exposed node labels list" + +// NodeLabels is the compiled form of the operator-level exposed-node-labels policy. An empty value +// disables propagation of any node label. +type NodeLabels []*regexp.Regexp + +// NewExposedNodeLabels compiles the given patterns into a NodeLabels policy. +func NewExposedNodeLabels(exposedNodeLabels []string) (NodeLabels, error) { + if len(exposedNodeLabels) == 0 { + return nil, nil + } + compiled := make([]*regexp.Regexp, len(exposedNodeLabels)) + for i, p := range exposedNodeLabels { + r, err := regexp.Compile(p) + if err != nil { + return nil, fmt.Errorf("exposed node label %q cannot be compiled as a regular expression: %w", p, err) + } + compiled[i] = r + } + return compiled, nil +} + +// IsAllowed returns whether the given node label is permitted by the policy. +func (n NodeLabels) IsAllowed(nodeLabel string) bool { + for _, r := range n { + if r.MatchString(nodeLabel) { + return true + } + } + return false +} + +// ValidateAnnotation checks that every label declared via the downward-node-labels annotation +// on the resource is allowed by the operator's exposed-node-labels policy. The check is a no-op +// when no annotation is set. +func ValidateAnnotation(annotations map[string]string, exposedNodeLabels NodeLabels) field.ErrorList { + var errs field.ErrorList + for _, nodeLabel := range nodelabels.FromAnnotations(annotations) { + if exposedNodeLabels.IsAllowed(nodeLabel) { + continue + } + errs = append(errs, field.Invalid( + field.NewPath("metadata").Child("annotations", nodelabels.DownwardNodeLabelsAnnotation), + nodeLabel, + NotAllowedNodesLabelMsg, + )) + } + return errs +} diff --git a/pkg/controller/common/nodelabels/nodelabels.go b/pkg/controller/common/nodelabels/nodelabels.go index 84ac6d31111..822d80686b3 100644 --- a/pkg/controller/common/nodelabels/nodelabels.go +++ b/pkg/controller/common/nodelabels/nodelabels.go @@ -16,6 +16,7 @@ import ( "go.elastic.co/apm/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" @@ -29,29 +30,36 @@ import ( // DownwardNodeLabelsAnnotation is re-exported for convenience. const DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation -// AnnotatePods copies the expected node labels as annotations on all Pods in the given namespace -// matching the given labelSelector. Missing node labels are reported as errors but do not stop the -// reconciliation of other Pods. -func AnnotatePods( - ctx context.Context, - c k8s.Client, - namespace string, - podLabelSelector map[string]string, - expectedLabels []string, - resourceName string, -) *reconciler.Results { +// AnnotationTarget is implemented by ECK custom resources whose managed Pods should have +// Kubernetes node labels copied to their annotations via AnnotatePods. Any ECK CR that already +// implements metav1.Object and GetIdentityLabels satisfies this interface once it exposes a +// DownwardNodeLabels accessor. +type AnnotationTarget interface { + metav1.Object + // DownwardNodeLabels returns the node labels expected to be copied as annotations on the + // Pods managed by the resource. An empty result disables node-label propagation. + DownwardNodeLabels() []string + // GetIdentityLabels returns the label set identifying Pods managed by the resource. + GetIdentityLabels() map[string]string +} + +// AnnotatePods copies the expected node labels as annotations on all Pods managed by the given +// target. Missing node labels are reported as errors but do not stop the reconciliation of +// other Pods. The call is a no-op when the target has no downward node labels configured. +func AnnotatePods(ctx context.Context, c k8s.Client, t AnnotationTarget) *reconciler.Results { span, ctx := apm.StartSpan(ctx, "annotate_pods_with_node_labels", tracing.SpanTypeApp) defer span.End() results := reconciler.NewResult(ctx) + expectedLabels := t.DownwardNodeLabels() if len(expectedLabels) == 0 { return results } - pods, err := k8s.PodsMatchingLabels(c, namespace, podLabelSelector) + pods, err := k8s.PodsMatchingLabels(c, t.GetNamespace(), t.GetIdentityLabels()) if err != nil { return results.WithError(err) } for _, pod := range pods { - results.WithError(annotatePod(ctx, c, pod, expectedLabels, resourceName)) + results.WithError(annotatePod(ctx, c, pod, expectedLabels, t.GetName())) } return results } diff --git a/pkg/controller/common/nodelabels/nodelabels_test.go b/pkg/controller/common/nodelabels/nodelabels_test.go index 0d474d8ddb3..4b4ecd15d1c 100644 --- a/pkg/controller/common/nodelabels/nodelabels_test.go +++ b/pkg/controller/common/nodelabels/nodelabels_test.go @@ -16,6 +16,16 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s" ) +// fakeTarget is a test implementation of AnnotationTarget. +type fakeTarget struct { + *corev1.Pod // embeds metav1.Object via ObjectMeta + labels []string + selector map[string]string +} + +func (f *fakeTarget) DownwardNodeLabels() []string { return f.labels } +func (f *fakeTarget) GetIdentityLabels() map[string]string { return f.selector } + func TestAnnotatePods(t *testing.T) { const namespace = "ns" podSelector := map[string]string{"app": "sample"} @@ -108,7 +118,12 @@ func TestAnnotatePods(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { c := k8s.NewFakeClient(tt.objects...) - results := AnnotatePods(context.Background(), c, namespace, podSelector, tt.expectedLabels, "sample") + target := &fakeTarget{ + Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "sample"}}, + labels: tt.expectedLabels, + selector: podSelector, + } + results := AnnotatePods(context.Background(), c, target) _, err := results.Aggregate() if tt.wantErrMsg != "" { assert.ErrorContains(t, err, tt.wantErrMsg) diff --git a/pkg/controller/common/webhook/resource_validator.go b/pkg/controller/common/webhook/resource_validator.go index 9dca2ef01da..26a51993a53 100644 --- a/pkg/controller/common/webhook/resource_validator.go +++ b/pkg/controller/common/webhook/resource_validator.go @@ -9,16 +9,46 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" "github.com/elastic/cloud-on-k8s/v3/pkg/utils/set" ) +// Validatable is the type constraint used by validation wrappers that need to access both the +// runtime and metadata interfaces of a resource. ECK CRDs satisfy it by embedding +// metav1.ObjectMeta and being registered with the runtime scheme. +type Validatable interface { + runtime.Object + metav1.Object +} + +// WithNodeLabelsValidation wraps a ValidateFunc with the operator's exposed-node-labels policy +// check, so that any value of the downward-node-labels annotation is validated consistently +// across resources. +func WithNodeLabelsValidation[T Validatable]( + validate ValidateFunc[T], + exposedNodeLabels commonnodelabels.NodeLabels, + groupKind schema.GroupKind, +) ValidateFunc[T] { + return func(obj T, old T) (admission.Warnings, error) { + warnings, err := validate(obj, old) + if err != nil { + return warnings, err + } + if errs := commonnodelabels.ValidateAnnotation(obj.GetAnnotations(), exposedNodeLabels); len(errs) > 0 { + return warnings, apierrors.NewInvalid(groupKind, obj.GetName(), errs) + } + return warnings, nil + } +} + // ValidateFunc is the per-resource validation callback. // obj is the object being validated, old is nil/zero on create. type ValidateFunc[T runtime.Object] func(obj T, old T) (admission.Warnings, error) diff --git a/pkg/controller/elasticsearch/validation/node_labels.go b/pkg/controller/elasticsearch/validation/node_labels.go index 33a1c3d81b4..b983c2c3f21 100644 --- a/pkg/controller/elasticsearch/validation/node_labels.go +++ b/pkg/controller/elasticsearch/validation/node_labels.go @@ -5,32 +5,14 @@ package validation import ( - "fmt" - "regexp" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" ) -type NodeLabels []*regexp.Regexp +// NodeLabels is an alias for the shared exposed-node-labels policy. It is kept here so that +// existing callers can continue to import esvalidation.NodeLabels. +type NodeLabels = commonnodelabels.NodeLabels +// NewExposedNodeLabels delegates to the shared exposed-node-labels constructor. func NewExposedNodeLabels(exposedNodeLabels []string) (NodeLabels, error) { - if len(exposedNodeLabels) == 0 { - return nil, nil - } - compiledNodeLabels := make([]*regexp.Regexp, len(exposedNodeLabels)) - for i, exposedNodeLabel := range exposedNodeLabels { - r, err := regexp.Compile(exposedNodeLabel) - if err != nil { - return nil, fmt.Errorf("exposed node label \"%s\" cannot be compiled as a regular expression: %w", exposedNodeLabel, err) - } - compiledNodeLabels[i] = r - } - return compiledNodeLabels, nil -} - -func (n NodeLabels) IsAllowed(nodeLabel string) bool { - for _, r := range n { - if r.MatchString(nodeLabel) { - return true - } - } - return false + return commonnodelabels.NewExposedNodeLabels(exposedNodeLabels) } diff --git a/pkg/controller/elasticsearch/validation/validations.go b/pkg/controller/elasticsearch/validation/validations.go index 8a43c6526ff..2e078823f1a 100644 --- a/pkg/controller/elasticsearch/validation/validations.go +++ b/pkg/controller/elasticsearch/validation/validations.go @@ -17,6 +17,7 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" esv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/elasticsearch/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" stackmon "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/stackmon/validations" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/elasticsearch/sset" @@ -116,26 +117,8 @@ func validations(ctx context.Context, checker license.Checker, exposedNodeLabels // Zone-awareness topology keys are only validated when the policy is configured (non-empty), so that // zone awareness works out of the box when no exposed-node-labels restriction is in place. func validNodeLabels(proposed esv1.Elasticsearch, exposedNodeLabels NodeLabels) field.ErrorList { - var errs field.ErrorList - annotationValue := "" - if proposed.Annotations != nil { - annotationValue = proposed.Annotations[esv1.DownwardNodeLabelsAnnotation] - } // firstly validate the downward-node-labels annotations - annotationLabels := esv1.ParseDownwardNodeLabels(annotationValue) - for nodeLabel := range annotationLabels { - if exposedNodeLabels.IsAllowed(nodeLabel) { - continue - } - errs = append( - errs, - field.Invalid( - field.NewPath("metadata").Child("annotations", esv1.DownwardNodeLabelsAnnotation), - nodeLabel, - notAllowedNodesLabelMsg, - ), - ) - } + errs := commonnodelabels.ValidateAnnotation(proposed.Annotations, exposedNodeLabels) // then validate the zone-awareness-derived topology keys if len(exposedNodeLabels) > 0 { diff --git a/pkg/controller/kibana/controller.go b/pkg/controller/kibana/controller.go index 0d3011cbd85..491992669f4 100644 --- a/pkg/controller/kibana/controller.go +++ b/pkg/controller/kibana/controller.go @@ -14,6 +14,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -28,6 +29,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/events" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/finalizer" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" @@ -217,6 +219,12 @@ func (r *ReconcileKibana) validate(ctx context.Context, kb *kbv1.Kibana) error { k8s.MaybeEmitErrorEvent(r.recorder, err, kb, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } + if errs := commonnodelabels.ValidateAnnotation(kb.Annotations, r.params.ExposedNodeLabels); len(errs) > 0 { + err := apierrors.NewInvalid(schema.GroupKind{Group: kbv1.GroupVersion.Group, Kind: kbv1.Kind}, kb.Name, errs) + ulog.FromContext(ctx).Error(err, "Validation failed") + k8s.MaybeEmitErrorEvent(r.recorder, err, kb, events.EventReasonValidation, events.EventActionValidation, err.Error()) + return tracing.CaptureError(vctx, err) + } for _, warning := range warnings { k8s.EmitEvent(r.recorder, kb, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } diff --git a/pkg/controller/kibana/driver.go b/pkg/controller/kibana/driver.go index 56542508ffc..e0841313f08 100644 --- a/pkg/controller/kibana/driver.go +++ b/pkg/controller/kibana/driver.go @@ -215,14 +215,7 @@ func (d *driver) Reconcile( // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the // reconciliation loop as we don't want to prevent other updates from being applied. - results.WithResults(nodelabels.AnnotatePods( - ctx, - d.K8sClient(), - kb.Namespace, - map[string]string{kblabel.KibanaNameLabelName: kb.Name}, - kb.DownwardNodeLabels(), - kb.Name, - )) + results.WithResults(nodelabels.AnnotatePods(ctx, d.K8sClient(), kb)) return results } diff --git a/pkg/controller/logstash/driver.go b/pkg/controller/logstash/driver.go index 30bb757120d..5a33ac70d3d 100644 --- a/pkg/controller/logstash/driver.go +++ b/pkg/controller/logstash/driver.go @@ -159,14 +159,7 @@ func internalReconcile(params Params) (*reconciler.Results, logstashv1alpha1.Log // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the // reconciliation loop as we don't want to prevent other updates from being applied. - statefulSetResults.WithResults(nodelabels.AnnotatePods( - params.Context, - params.Client, - params.Logstash.Namespace, - map[string]string{labels.NameLabelName: params.Logstash.Name}, - params.Logstash.DownwardNodeLabels(), - params.Logstash.Name, - )) + statefulSetResults.WithResults(nodelabels.AnnotatePods(params.Context, params.Client, ¶ms.Logstash)) return statefulSetResults, status } diff --git a/pkg/controller/logstash/logstash_controller.go b/pkg/controller/logstash/logstash_controller.go index e935262b785..86c6257dfcc 100644 --- a/pkg/controller/logstash/logstash_controller.go +++ b/pkg/controller/logstash/logstash_controller.go @@ -214,7 +214,7 @@ func (r *ReconcileLogstash) validate(ctx context.Context, logstash logstashv1alp defer tracing.Span(&ctx)() // Run create validations only as update validations require old object which we don't have here. - warnings, err := validation.ValidateLogstash(&logstash) + warnings, err := validation.ValidateLogstash(&logstash, r.ExposedNodeLabels) if err != nil { ulog.FromContext(ctx).Error(err, "Validation failed") k8s.MaybeEmitErrorEvent(r.recorder, err, &logstash, events.EventReasonValidation, events.EventActionValidation, err.Error()) diff --git a/pkg/controller/logstash/validation/webhook.go b/pkg/controller/logstash/validation/webhook.go index 464e2139fac..d32ec9583fa 100644 --- a/pkg/controller/logstash/validation/webhook.go +++ b/pkg/controller/logstash/validation/webhook.go @@ -15,6 +15,7 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" lsv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/logstash/v1alpha1" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" commonwebhook "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/webhook" "github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s" ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" @@ -29,10 +30,11 @@ const ( var lslog = ulog.Log.WithName("ls-validation") // RegisterWebhook registers the Logstash validating webhook. -func RegisterWebhook(mgr ctrl.Manager, validateStorageClass bool, managedNamespaces []string) { +func RegisterWebhook(mgr ctrl.Manager, validateStorageClass bool, exposedNodeLabels commonnodelabels.NodeLabels, managedNamespaces []string) { inner := &validator{ client: mgr.GetClient(), validateStorageClass: validateStorageClass, + exposedNodeLabels: exposedNodeLabels, } // Logstash has no license-dependent validation, so we pass nil here. v := commonwebhook.NewResourceValidator[*lsv1alpha1.Logstash](nil, managedNamespaces, inner) @@ -44,11 +46,12 @@ func RegisterWebhook(mgr ctrl.Manager, validateStorageClass bool, managedNamespa type validator struct { client k8s.Client validateStorageClass bool + exposedNodeLabels commonnodelabels.NodeLabels } func (v *validator) ValidateCreate(_ context.Context, ls *lsv1alpha1.Logstash) (admission.Warnings, error) { lslog.V(1).Info("validate create", "name", ls.Name) - return ValidateLogstash(ls) + return ValidateLogstash(ls, v.exposedNodeLabels) } func (v *validator) ValidateUpdate(ctx context.Context, oldObj, newObj *lsv1alpha1.Logstash) (admission.Warnings, error) { @@ -56,7 +59,7 @@ func (v *validator) ValidateUpdate(ctx context.Context, oldObj, newObj *lsv1alph // Match Elasticsearch: run full validation on the new object first so warnings are collected before // update-only checks; when update-only checks fail, return those errors but keep prior warnings. - warnings, valErr := ValidateLogstash(newObj) + warnings, valErr := ValidateLogstash(newObj, v.exposedNodeLabels) var errs field.ErrorList for _, val := range updateValidations(ctx, v.client, v.validateStorageClass) { @@ -78,7 +81,7 @@ func (v *validator) ValidateDelete(_ context.Context, _ *lsv1alpha1.Logstash) (a // ValidateLogstash validates a Logstash instance against a set of validation funcs. // Returns any admission warnings plus an error if validation fails. -func ValidateLogstash(ls *lsv1alpha1.Logstash) (admission.Warnings, error) { +func ValidateLogstash(ls *lsv1alpha1.Logstash, exposedNodeLabels commonnodelabels.NodeLabels) (admission.Warnings, error) { var warnings admission.Warnings // CheckDeprecatedStackVersion's second return is field.ErrorList; it is always nil // (parse/unsupported-version failures are handled by validations(), not here). @@ -95,6 +98,7 @@ func ValidateLogstash(ls *lsv1alpha1.Logstash) (admission.Warnings, error) { warnings = append(warnings, resourcesWarning) } errs := check(ls, validations()) + errs = append(errs, commonnodelabels.ValidateAnnotation(ls.Annotations, exposedNodeLabels)...) if len(errs) > 0 { return warnings, apierrors.NewInvalid( schema.GroupKind{Group: "logstash.k8s.elastic.co", Kind: lsv1alpha1.Kind}, diff --git a/pkg/controller/maps/controller.go b/pkg/controller/maps/controller.go index 4725b24ec7c..ec41cce5a0c 100644 --- a/pkg/controller/maps/controller.go +++ b/pkg/controller/maps/controller.go @@ -16,6 +16,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -35,6 +36,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/events" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" @@ -276,6 +278,10 @@ func (r *ReconcileMapsServer) doReconcile(ctx context.Context, ems emsv1alpha1.E return results.WithError(fmt.Errorf("calculating status: %w", err)), status } + // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the + // reconciliation loop as we don't want to prevent other updates from being applied. + results.WithResults(commonnodelabels.AnnotatePods(ctx, r.K8sClient(), &ems)) + return results, status } @@ -295,6 +301,12 @@ func (r *ReconcileMapsServer) validate(ctx context.Context, ems emsv1alpha1.Elas k8s.MaybeEmitErrorEvent(r.recorder, err, &ems, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } + if errs := commonnodelabels.ValidateAnnotation(ems.Annotations, r.ExposedNodeLabels); len(errs) > 0 { + err := apierrors.NewInvalid(schema.GroupKind{Group: emsv1alpha1.GroupVersion.Group, Kind: emsv1alpha1.Kind}, ems.Name, errs) + ulog.FromContext(ctx).Error(err, "Validation failed") + k8s.MaybeEmitErrorEvent(r.recorder, err, &ems, events.EventReasonValidation, events.EventActionValidation, err.Error()) + return tracing.CaptureError(vctx, err) + } for _, warning := range warnings { k8s.EmitEvent(r.recorder, &ems, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } @@ -330,6 +342,12 @@ func buildConfigHash(c k8s.Client, ems emsv1alpha1.ElasticMapsServer, configSecr // - in the Elastic Maps Server configuration file content _, _ = configHash.Write(configSecret.Data[ConfigFilename]) + // Changes to the downward-node-labels annotation must roll the Elastic Maps Server Pods so the new + // annotations are re-applied on scheduling. + if ems.HasDownwardNodeLabels() { + _, _ = configHash.Write([]byte(ems.Annotations[emsv1alpha1.DownwardNodeLabelsAnnotation])) + } + // - in the Elastic Maps Server TLS certificates if ems.Spec.HTTP.TLS.Enabled() { var tlsCertSecret corev1.Secret diff --git a/pkg/controller/maps/pod.go b/pkg/controller/maps/pod.go index 395b05b347c..b0c78808ca6 100644 --- a/pkg/controller/maps/pod.go +++ b/pkg/controller/maps/pod.go @@ -14,6 +14,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/defaults" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/volume" ) @@ -115,6 +116,20 @@ func newPodSpec(ems emsv1alpha1.ElasticMapsServer, configHash string, meta metad } builder = withHTTPCertsVolume(builder, ems) + // Block Pod start on the operator patching the expected node-label annotations onto the + // Pod, so the Elastic Maps Server container does not start while those annotations are missing + // from the downward-API annotations file. + if ems.HasDownwardNodeLabels() { + downwardAPIVolume := commonnodelabels.DownwardAPIVolume() + waitInit, err := commonnodelabels.WaitForAnnotationsInitContainer(ems.DownwardNodeLabels()) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + builder = builder. + WithVolumes(downwardAPIVolume.Volume()). + WithInitContainers(waitInit) + } + esAssocConf, err := ems.AssociationConf() if err != nil { return corev1.PodTemplateSpec{}, err diff --git a/pkg/controller/packageregistry/controller.go b/pkg/controller/packageregistry/controller.go index 62486dfcdc8..056ca1e1ed4 100644 --- a/pkg/controller/packageregistry/controller.go +++ b/pkg/controller/packageregistry/controller.go @@ -15,6 +15,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -31,6 +32,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/driver" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/events" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" @@ -244,6 +246,10 @@ func (r *ReconcilePackageRegistry) doReconcile(ctx context.Context, epr eprv1alp } status.DeploymentStatus = deploymentStatus + // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the + // reconciliation loop as we don't want to prevent other updates from being applied. + results.WithResults(commonnodelabels.AnnotatePods(ctx, r.K8sClient(), &epr)) + return results, status } @@ -263,6 +269,12 @@ func (r *ReconcilePackageRegistry) validate(ctx context.Context, epr eprv1alpha1 k8s.MaybeEmitErrorEvent(r.recorder, err, &epr, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } + if errs := commonnodelabels.ValidateAnnotation(epr.Annotations, r.ExposedNodeLabels); len(errs) > 0 { + err := apierrors.NewInvalid(schema.GroupKind{Group: eprv1alpha1.GroupVersion.Group, Kind: eprv1alpha1.Kind}, epr.Name, errs) + ulog.FromContext(ctx).Error(err, "Validation failed") + k8s.MaybeEmitErrorEvent(r.recorder, err, &epr, events.EventReasonValidation, events.EventActionValidation, err.Error()) + return tracing.CaptureError(vctx, err) + } for _, warning := range warnings { k8s.EmitEvent(r.recorder, &epr, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } @@ -305,6 +317,12 @@ func buildConfigHash(epr eprv1alpha1.PackageRegistry, configSecret corev1.Secret } } + // Changes to the downward-node-labels annotation must roll the Package Registry Pods so the new + // annotations are re-applied on scheduling. + if epr.HasDownwardNodeLabels() { + _, _ = configHash.Write([]byte(epr.Annotations[eprv1alpha1.DownwardNodeLabelsAnnotation])) + } + return fmt.Sprint(configHash.Sum32()) } diff --git a/pkg/controller/packageregistry/pod.go b/pkg/controller/packageregistry/pod.go index 311b35424f4..7d22baec58b 100644 --- a/pkg/controller/packageregistry/pod.go +++ b/pkg/controller/packageregistry/pod.go @@ -17,6 +17,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/container" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/defaults" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" ) @@ -129,6 +130,20 @@ func newPodSpec(epr eprv1alpha1.PackageRegistry, configHash string, meta metadat // Add HTTP certificates volume if TLS is enabled builder = withHTTPCertsVolume(builder, epr) + // Block Pod start on the operator patching the expected node-label annotations onto the + // Pod, so the Package Registry container does not start while those annotations are missing + // from the downward-API annotations file. + if epr.HasDownwardNodeLabels() { + downwardAPIVolume := commonnodelabels.DownwardAPIVolume() + waitInit, err := commonnodelabels.WaitForAnnotationsInitContainer(epr.DownwardNodeLabels()) + if err != nil { + return corev1.PodTemplateSpec{}, err + } + builder = builder. + WithVolumes(downwardAPIVolume.Volume()). + WithInitContainers(waitInit) + } + return builder.PodTemplate, nil } From 54060226eb4051b1627fe13aa6853e58aa5e17a8 Mon Sep 17 00:00:00 2001 From: Joel Teo Date: Thu, 25 Jun 2026 19:51:46 +0800 Subject: [PATCH 4/4] Address review feedback: relocate node-labels API and unify validation - Move DownwardNodeLabelsAnnotation and parsing helpers to pkg/apis/common/v1 as the canonical definition; drop the per-CRD re-exports and delete pkg/utils/nodelabels. - Remove the ES-specific volume dependency from the common nodelabels init container; derive the annotations-file path via commonvolume. - Align the wait script grep with the Elasticsearch prepare-fs script (BRE). - Migrate the Elasticsearch driver to nodelabels.AnnotatePods and delete the duplicated shared/node_labels.go implementation. - Introduce per-type RegisterWebhook for APM Server, Beat, Kibana, Maps and Package Registry so a single validation function enforces the exposed-node-labels policy on both the webhook and reconciler paths; remove the WithNodeLabelsValidation wrapper. - Call WithInitContainerDefaults unconditionally in the Agent pod template. - Fix gofmt formatting in nodelabels_test.go. --- cmd/manager/validation.go | 34 ++- pkg/apis/agent/v1alpha1/agent_types.go | 6 +- pkg/apis/apm/v1/apmserver_types.go | 6 +- pkg/apis/beat/v1beta1/beat_types.go | 6 +- .../common/v1/node_labels.go} | 21 +- .../common/v1/node_labels_test.go} | 14 +- .../elasticsearch/v1/elasticsearch_types.go | 2 +- pkg/apis/kibana/v1/kibana_types.go | 6 +- pkg/apis/logstash/v1alpha1/logstash_types.go | 7 +- pkg/apis/maps/v1alpha1/maps_types.go | 6 +- .../packageregistry/v1alpha1/epr_types.go | 6 +- pkg/controller/agent/pod.go | 7 +- pkg/controller/apmserver/controller.go | 9 +- pkg/controller/apmserver/deployment.go | 3 +- pkg/controller/apmserver/webhook.go | 61 +++++ pkg/controller/beat/common/pod.go | 2 +- pkg/controller/beat/controller.go | 10 +- pkg/controller/beat/webhook.go | 61 +++++ pkg/controller/common/nodelabels/exposed.go | 6 +- .../common/nodelabels/initcontainer.go | 6 +- .../common/nodelabels/initcontainer_test.go | 5 +- .../common/nodelabels/nodelabels.go | 4 - .../common/nodelabels/nodelabels_test.go | 4 +- pkg/controller/common/volume/downward_api.go | 8 + .../common/webhook/resource_validator.go | 30 --- .../driver/shared/node_labels.go | 116 --------- .../driver/shared/node_labels_test.go | 238 ------------------ .../elasticsearch/driver/shared/reconcile.go | 3 +- pkg/controller/kibana/controller.go | 10 +- pkg/controller/kibana/driver.go | 3 +- pkg/controller/kibana/webhook.go | 61 +++++ pkg/controller/logstash/pod.go | 2 +- pkg/controller/maps/controller.go | 12 +- pkg/controller/maps/webhook.go | 61 +++++ pkg/controller/packageregistry/controller.go | 12 +- pkg/controller/packageregistry/webhook.go | 61 +++++ 36 files changed, 383 insertions(+), 526 deletions(-) rename pkg/{utils/nodelabels/nodelabels.go => apis/common/v1/node_labels.go} (55%) rename pkg/{utils/nodelabels/nodelabels_test.go => apis/common/v1/node_labels_test.go} (66%) create mode 100644 pkg/controller/apmserver/webhook.go create mode 100644 pkg/controller/beat/webhook.go delete mode 100644 pkg/controller/elasticsearch/driver/shared/node_labels.go delete mode 100644 pkg/controller/elasticsearch/driver/shared/node_labels_test.go create mode 100644 pkg/controller/kibana/webhook.go create mode 100644 pkg/controller/maps/webhook.go create mode 100644 pkg/controller/packageregistry/webhook.go diff --git a/cmd/manager/validation.go b/cmd/manager/validation.go index 7c331e1dd68..0d282bc2772 100644 --- a/cmd/manager/validation.go +++ b/cmd/manager/validation.go @@ -15,32 +15,31 @@ import ( "github.com/spf13/viper" "go.elastic.co/apm/v2" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" "sigs.k8s.io/controller-runtime/pkg/manager" - apmv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/apm/v1" apmv1beta1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/apm/v1beta1" - beatv1beta1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/beat/v1beta1" esv1beta1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/elasticsearch/v1beta1" entv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/enterprisesearch/v1" entv1beta1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/enterprisesearch/v1beta1" - kbv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/kibana/v1" kbv1beta1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/kibana/v1beta1" - emsv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/maps/v1alpha1" - eprv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/packageregistry/v1alpha1" policyv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/stackconfigpolicy/v1alpha1" agentcontroller "github.com/elastic/cloud-on-k8s/v3/pkg/controller/agent" + apmservercontroller "github.com/elastic/cloud-on-k8s/v3/pkg/controller/apmserver" autoopsvalidation "github.com/elastic/cloud-on-k8s/v3/pkg/controller/autoops/validation" esavalidation "github.com/elastic/cloud-on-k8s/v3/pkg/controller/autoscaling/elasticsearch/validation" + beatcontroller "github.com/elastic/cloud-on-k8s/v3/pkg/controller/beat" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/certificates" commonlicense "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" commonwebhook "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/webhook" esvalidation "github.com/elastic/cloud-on-k8s/v3/pkg/controller/elasticsearch/validation" + kibanacontroller "github.com/elastic/cloud-on-k8s/v3/pkg/controller/kibana" lsvalidation "github.com/elastic/cloud-on-k8s/v3/pkg/controller/logstash/validation" + mapscontroller "github.com/elastic/cloud-on-k8s/v3/pkg/controller/maps" + eprcontroller "github.com/elastic/cloud-on-k8s/v3/pkg/controller/packageregistry" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/webhook" ) @@ -88,31 +87,28 @@ func setupWebhook( checker := commonlicense.NewLicenseChecker(mgr.GetClient(), params.OperatorNamespace) // setup webhooks for supported types - commonwebhook.RegisterResourceWebhook(mgr, apmv1.WebhookPath, checker, managedNamespaces, - commonwebhook.WithNodeLabelsValidation(apmv1.Validate, exposedNodeLabels, schema.GroupKind{Group: apmv1.GroupVersion.Group, Kind: apmv1.Kind}), "APM Server") commonwebhook.RegisterResourceWebhook(mgr, apmv1beta1.WebhookPath, checker, managedNamespaces, apmv1beta1.Validate, "APM Server") - commonwebhook.RegisterResourceWebhook(mgr, beatv1beta1.WebhookPath, checker, managedNamespaces, - commonwebhook.WithNodeLabelsValidation(beatv1beta1.Validate, exposedNodeLabels, schema.GroupKind{Group: beatv1beta1.GroupVersion.Group, Kind: beatv1beta1.Kind}), "Beat") commonwebhook.RegisterResourceWebhook(mgr, entv1.WebhookPath, checker, managedNamespaces, entv1.Validate, "Enterprise Search") commonwebhook.RegisterResourceWebhook(mgr, entv1beta1.WebhookPath, checker, managedNamespaces, entv1beta1.Validate, "Enterprise Search") commonwebhook.RegisterResourceWebhook(mgr, esv1beta1.WebhookPath, checker, managedNamespaces, esv1beta1.Validate, "Elasticsearch") - commonwebhook.RegisterResourceWebhook(mgr, kbv1.WebhookPath, checker, managedNamespaces, - commonwebhook.WithNodeLabelsValidation(kbv1.Validate, exposedNodeLabels, schema.GroupKind{Group: kbv1.GroupVersion.Group, Kind: kbv1.Kind}), "Kibana") commonwebhook.RegisterResourceWebhook(mgr, kbv1beta1.WebhookPath, checker, managedNamespaces, kbv1beta1.Validate, "Kibana") - commonwebhook.RegisterResourceWebhook(mgr, emsv1alpha1.WebhookPath, checker, managedNamespaces, - commonwebhook.WithNodeLabelsValidation(emsv1alpha1.Validate, exposedNodeLabels, schema.GroupKind{Group: emsv1alpha1.GroupVersion.Group, Kind: emsv1alpha1.Kind}), "Elastic Maps Server") - commonwebhook.RegisterResourceWebhook(mgr, eprv1alpha1.WebhookPath, checker, managedNamespaces, - commonwebhook.WithNodeLabelsValidation(eprv1alpha1.Validate, exposedNodeLabels, schema.GroupKind{Group: eprv1alpha1.GroupVersion.Group, Kind: eprv1alpha1.Kind}), "Package Registry") commonwebhook.RegisterResourceWebhook(mgr, policyv1alpha1.WebhookPath, checker, managedNamespaces, policyv1alpha1.Validate, "Stack Config Policy") - // Logstash, Elasticsearch v1, ElasticsearchAutoscaling, and AutoOps validating webhooks are wired up - // separately so their validators can use the API client and/or embed license checks. Elasticsearch - // v1beta1 remains in the RegisterResourceWebhook list above because it only needs a ValidateFunc. + // Types that support the exposed-node-labels policy register their own webhook so that a single + // validation function enforces the policy on both the webhook and reconciler paths. The same is + // true for Elasticsearch v1, ElasticsearchAutoscaling, and AutoOps, whose validators additionally + // use the API client and/or embed license checks. Elasticsearch v1beta1 and the other types above + // remain in the RegisterResourceWebhook list because they only need a ValidateFunc. esvalidation.RegisterWebhook(mgr, params.ValidateStorageClass, exposedNodeLabels, checker, managedNamespaces) esavalidation.RegisterWebhook(mgr, params.ValidateStorageClass, checker, managedNamespaces) lsvalidation.RegisterWebhook(mgr, params.ValidateStorageClass, exposedNodeLabels, managedNamespaces) autoopsvalidation.RegisterWebhook(mgr, checker, managedNamespaces) agentcontroller.RegisterWebhook(mgr, checker, exposedNodeLabels, managedNamespaces) + apmservercontroller.RegisterWebhook(mgr, checker, exposedNodeLabels, managedNamespaces) + beatcontroller.RegisterWebhook(mgr, checker, exposedNodeLabels, managedNamespaces) + kibanacontroller.RegisterWebhook(mgr, checker, exposedNodeLabels, managedNamespaces) + mapscontroller.RegisterWebhook(mgr, checker, exposedNodeLabels, managedNamespaces) + eprcontroller.RegisterWebhook(mgr, checker, exposedNodeLabels, managedNamespaces) // wait for the secret to be populated in the local filesystem before returning interval := time.Second * 1 diff --git a/pkg/apis/agent/v1alpha1/agent_types.go b/pkg/apis/agent/v1alpha1/agent_types.go index 6a2a4f99626..a6ed6b74da0 100644 --- a/pkg/apis/agent/v1alpha1/agent_types.go +++ b/pkg/apis/agent/v1alpha1/agent_types.go @@ -14,7 +14,6 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -25,9 +24,6 @@ const ( AgentContainerName = "agent" // FleetServerServiceAccount is the Elasticsearch service account to be used to authenticate. FleetServerServiceAccount commonv1.ServiceAccountName = "fleet-server" - // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels - // to be set as annotations on the Elastic Agent Pods. - DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // AgentSpec defines the desired state of the Agent @@ -330,7 +326,7 @@ func (a *Agent) IsMarkedForDeletion() bool { // DownwardNodeLabels returns the node labels to copy as annotations on the Agent Pods, // as declared via the DownwardNodeLabelsAnnotation annotation. func (a *Agent) DownwardNodeLabels() []string { - return nodelabels.FromAnnotations(a.Annotations) + return commonv1.DownwardNodeLabelsFromAnnotations(a.Annotations) } // HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Agent Pods. diff --git a/pkg/apis/apm/v1/apmserver_types.go b/pkg/apis/apm/v1/apmserver_types.go index eb53b96febc..08b25f091d0 100644 --- a/pkg/apis/apm/v1/apmserver_types.go +++ b/pkg/apis/apm/v1/apmserver_types.go @@ -12,7 +12,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -20,9 +19,6 @@ const ( // Kind is inferred from the struct name using reflection in scheme.AddKnownTypes() // we duplicate it as a constant here for practical purposes. Kind = "ApmServer" - // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels - // to be set as annotations on the APM Server Pods. - DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // ApmServerSpec holds the specification of an APM Server. @@ -135,7 +131,7 @@ func (as *ApmServer) IsMarkedForDeletion() bool { // DownwardNodeLabels returns the node labels to copy as annotations on the APM Server Pods, // as declared via the DownwardNodeLabelsAnnotation annotation. func (as *ApmServer) DownwardNodeLabels() []string { - return nodelabels.FromAnnotations(as.Annotations) + return commonv1.DownwardNodeLabelsFromAnnotations(as.Annotations) } // HasDownwardNodeLabels returns true if node labels are expected to be propagated to the APM Server Pods. diff --git a/pkg/apis/beat/v1beta1/beat_types.go b/pkg/apis/beat/v1beta1/beat_types.go index c1cd907d555..7c55ad1e1d1 100644 --- a/pkg/apis/beat/v1beta1/beat_types.go +++ b/pkg/apis/beat/v1beta1/beat_types.go @@ -13,16 +13,12 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/stackmon/monitoring" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( // Kind is inferred from the struct name using reflection in scheme.AddKnownTypes() // we duplicate it as a constant here for practical purposes. Kind = "Beat" - // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels - // to be set as annotations on the Beat Pods. - DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) var ( @@ -288,7 +284,7 @@ func (b *Beat) IsMarkedForDeletion() bool { // DownwardNodeLabels returns the node labels to copy as annotations on the Beat Pods, // as declared via the DownwardNodeLabelsAnnotation annotation. func (b *Beat) DownwardNodeLabels() []string { - return nodelabels.FromAnnotations(b.Annotations) + return commonv1.DownwardNodeLabelsFromAnnotations(b.Annotations) } // HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Beat Pods. diff --git a/pkg/utils/nodelabels/nodelabels.go b/pkg/apis/common/v1/node_labels.go similarity index 55% rename from pkg/utils/nodelabels/nodelabels.go rename to pkg/apis/common/v1/node_labels.go index daf7f947cce..332a474f47b 100644 --- a/pkg/utils/nodelabels/nodelabels.go +++ b/pkg/apis/common/v1/node_labels.go @@ -2,10 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -// Package nodelabels provides the constant and parsing helpers used by the ECK resources to -// declare the set of Kubernetes node labels that must be copied to the annotations of the Pods -// managed by a given resource. -package nodelabels +package v1 import ( "strings" @@ -14,12 +11,13 @@ import ( ) // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels to -// be set as annotations on the Pods managed by an ECK resource. +// be set as annotations on the Pods managed by an ECK resource. It is a user-facing API annotation +// shared by all ECK resource types, so the canonical definition lives here. const DownwardNodeLabelsAnnotation = "eck.k8s.elastic.co/downward-node-labels" -// Parse normalizes a comma-separated node labels annotation value into a sorted, deduplicated -// slice. An empty or whitespace-only value returns nil. -func Parse(annotationValue string) []string { +// ParseDownwardNodeLabels normalizes a comma-separated node labels annotation value into a sorted, +// deduplicated slice. An empty or whitespace-only value returns nil. +func ParseDownwardNodeLabels(annotationValue string) []string { labels := set.Make() for label := range strings.SplitSeq(annotationValue, ",") { label = strings.TrimSpace(label) @@ -34,7 +32,8 @@ func Parse(annotationValue string) []string { return labels.AsSortedSlice() } -// FromAnnotations returns the list of downward node labels declared via DownwardNodeLabelsAnnotation. -func FromAnnotations(annotations map[string]string) []string { - return Parse(annotations[DownwardNodeLabelsAnnotation]) +// DownwardNodeLabelsFromAnnotations returns the list of downward node labels declared via the +// DownwardNodeLabelsAnnotation annotation. +func DownwardNodeLabelsFromAnnotations(annotations map[string]string) []string { + return ParseDownwardNodeLabels(annotations[DownwardNodeLabelsAnnotation]) } diff --git a/pkg/utils/nodelabels/nodelabels_test.go b/pkg/apis/common/v1/node_labels_test.go similarity index 66% rename from pkg/utils/nodelabels/nodelabels_test.go rename to pkg/apis/common/v1/node_labels_test.go index 1250a3ee930..800abd27b4d 100644 --- a/pkg/utils/nodelabels/nodelabels_test.go +++ b/pkg/apis/common/v1/node_labels_test.go @@ -2,7 +2,7 @@ // or more contributor license agreements. Licensed under the Elastic License 2.0; // you may not use this file except in compliance with the Elastic License 2.0. -package nodelabels +package v1 import ( "testing" @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestParse(t *testing.T) { +func TestParseDownwardNodeLabels(t *testing.T) { tests := []struct { name, input string want []string @@ -26,17 +26,17 @@ func TestParse(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - assert.Equal(t, tt.want, Parse(tt.input)) + assert.Equal(t, tt.want, ParseDownwardNodeLabels(tt.input)) }) } } -func TestFromAnnotations(t *testing.T) { - assert.Nil(t, FromAnnotations(nil)) - assert.Nil(t, FromAnnotations(map[string]string{"other": "value"})) +func TestDownwardNodeLabelsFromAnnotations(t *testing.T) { + assert.Nil(t, DownwardNodeLabelsFromAnnotations(nil)) + assert.Nil(t, DownwardNodeLabelsFromAnnotations(map[string]string{"other": "value"})) assert.Equal( t, []string{"topology.kubernetes.io/region", "topology.kubernetes.io/zone"}, - FromAnnotations(map[string]string{DownwardNodeLabelsAnnotation: "topology.kubernetes.io/zone,topology.kubernetes.io/region"}), + DownwardNodeLabelsFromAnnotations(map[string]string{DownwardNodeLabelsAnnotation: "topology.kubernetes.io/zone,topology.kubernetes.io/region"}), ) } diff --git a/pkg/apis/elasticsearch/v1/elasticsearch_types.go b/pkg/apis/elasticsearch/v1/elasticsearch_types.go index 90d0b7e90c1..458be677ba5 100644 --- a/pkg/apis/elasticsearch/v1/elasticsearch_types.go +++ b/pkg/apis/elasticsearch/v1/elasticsearch_types.go @@ -33,7 +33,7 @@ const ( // eck.k8s.elastic.co/disable-upgrade-predicates="if_yellow_only_restart_upgrading_nodes_with_unassigned_replicas" DisableUpgradePredicatesAnnotation = "eck.k8s.elastic.co/disable-upgrade-predicates" // DownwardNodeLabelsAnnotation holds an optional list of expected node labels to be set as annotations on the Elasticsearch Pods. - DownwardNodeLabelsAnnotation = "eck.k8s.elastic.co/downward-node-labels" + DownwardNodeLabelsAnnotation = commonv1.DownwardNodeLabelsAnnotation // SuspendAnnotation allows users to annotate the Elasticsearch resource with the names of Pods they want to suspend // for debugging purposes. SuspendAnnotation = "eck.k8s.elastic.co/suspend" diff --git a/pkg/apis/kibana/v1/kibana_types.go b/pkg/apis/kibana/v1/kibana_types.go index aacbd339a8c..cb6e9df9937 100644 --- a/pkg/apis/kibana/v1/kibana_types.go +++ b/pkg/apis/kibana/v1/kibana_types.go @@ -13,7 +13,6 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -23,9 +22,6 @@ const ( Kind = "Kibana" // KibanaServiceAccount is the Elasticsearch service account to be used to authenticate. KibanaServiceAccount commonv1.ServiceAccountName = "kibana" - // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels - // to be set as annotations on the Kibana Pods. - DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // +kubebuilder:object:root=true @@ -157,7 +153,7 @@ func (k *Kibana) IsMarkedForDeletion() bool { // DownwardNodeLabels returns the node labels to copy as annotations on the Kibana Pods, // as declared via the DownwardNodeLabelsAnnotation annotation. func (k *Kibana) DownwardNodeLabels() []string { - return nodelabels.FromAnnotations(k.Annotations) + return commonv1.DownwardNodeLabelsFromAnnotations(k.Annotations) } // HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Kibana Pods. diff --git a/pkg/apis/logstash/v1alpha1/logstash_types.go b/pkg/apis/logstash/v1alpha1/logstash_types.go index 54ee37aab34..adaece37cb0 100644 --- a/pkg/apis/logstash/v1alpha1/logstash_types.go +++ b/pkg/apis/logstash/v1alpha1/logstash_types.go @@ -13,7 +13,6 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/hash" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) type LogstashHealth string @@ -36,10 +35,6 @@ const ( // 1) all Pods are Ready, and // 2) any associations are configured and established LogstashGreenHealth LogstashHealth = "green" - - // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels - // to be set as annotations on the Logstash Pods. - DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // LogstashSpec defines the desired state of Logstash @@ -228,7 +223,7 @@ func (l *Logstash) IsMarkedForDeletion() bool { // DownwardNodeLabels returns the node labels to copy as annotations on the Logstash Pods, // as declared via the DownwardNodeLabelsAnnotation annotation. func (l *Logstash) DownwardNodeLabels() []string { - return nodelabels.FromAnnotations(l.Annotations) + return commonv1.DownwardNodeLabelsFromAnnotations(l.Annotations) } // HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Logstash Pods. diff --git a/pkg/apis/maps/v1alpha1/maps_types.go b/pkg/apis/maps/v1alpha1/maps_types.go index 062d7edbefb..f5d9e1bba56 100644 --- a/pkg/apis/maps/v1alpha1/maps_types.go +++ b/pkg/apis/maps/v1alpha1/maps_types.go @@ -11,7 +11,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -19,9 +18,6 @@ const ( // Kind is inferred from the struct name using reflection in scheme.AddKnownTypes() // we duplicate it as a constant here for practical purposes. Kind = "ElasticMapsServer" - // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels - // to be set as annotations on the Elastic Maps Server Pods. - DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // MapsSpec holds the specification of an Elastic Maps Server instance. @@ -91,7 +87,7 @@ func (m *ElasticMapsServer) IsMarkedForDeletion() bool { // DownwardNodeLabels returns the node labels to copy as annotations on the Elastic Maps Server Pods, // as declared via the DownwardNodeLabelsAnnotation annotation. func (m *ElasticMapsServer) DownwardNodeLabels() []string { - return nodelabels.FromAnnotations(m.Annotations) + return commonv1.DownwardNodeLabelsFromAnnotations(m.Annotations) } // HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Elastic Maps Server Pods. diff --git a/pkg/apis/packageregistry/v1alpha1/epr_types.go b/pkg/apis/packageregistry/v1alpha1/epr_types.go index d8d401e05b2..c32687acdfd 100644 --- a/pkg/apis/packageregistry/v1alpha1/epr_types.go +++ b/pkg/apis/packageregistry/v1alpha1/epr_types.go @@ -10,7 +10,6 @@ import ( commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" common_name "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/name" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) const ( @@ -18,9 +17,6 @@ const ( // Kind is inferred from the struct name using reflection in scheme.AddKnownTypes() // we duplicate it as a constant here for practical purposes. Kind = "PackageRegistry" - // DownwardNodeLabelsAnnotation holds an optional comma-separated list of expected node labels - // to be set as annotations on the Elastic Package Registry Pods. - DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation ) // Namer is a Namer that is configured with the defaults for resources related to a Package Registry resource. @@ -102,7 +98,7 @@ func (m *PackageRegistry) IsMarkedForDeletion() bool { // DownwardNodeLabels returns the node labels to copy as annotations on the Elastic Package Registry Pods, // as declared via the DownwardNodeLabelsAnnotation annotation. func (m *PackageRegistry) DownwardNodeLabels() []string { - return nodelabels.FromAnnotations(m.Annotations) + return commonv1.DownwardNodeLabelsFromAnnotations(m.Annotations) } // HasDownwardNodeLabels returns true if node labels are expected to be propagated to the Elastic Package Registry Pods. diff --git a/pkg/controller/agent/pod.go b/pkg/controller/agent/pod.go index ba83b5f12ba..4a98d853e51 100644 --- a/pkg/controller/agent/pod.go +++ b/pkg/controller/agent/pod.go @@ -213,7 +213,7 @@ func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret // Changes to the downward-node-labels annotation must roll the Agent Pods so the new annotations // are re-applied on scheduling. if params.Agent.HasDownwardNodeLabels() { - _, _ = configHash.Write([]byte(params.Agent.Annotations[agentv1alpha1.DownwardNodeLabelsAnnotation])) + _, _ = configHash.Write([]byte(params.Agent.Annotations[commonv1.DownwardNodeLabelsAnnotation])) } podMeta := params.Meta.Merge(metadata.Metadata{ @@ -245,11 +245,10 @@ func buildPodTemplate(params Params, fleetCerts *certificates.CertificatesSecret } builder = builder. WithVolumes(downwardAPIVolume.Volume()). - WithInitContainers(waitInit). - WithInitContainerDefaults() + WithInitContainers(waitInit) } - return builder.PodTemplate, nil + return builder.WithInitContainerDefaults().PodTemplate, nil } func fleetConfigPath(v version.Version) string { diff --git a/pkg/controller/apmserver/controller.go b/pkg/controller/apmserver/controller.go index da498299321..ad69b3df2d9 100644 --- a/pkg/controller/apmserver/controller.go +++ b/pkg/controller/apmserver/controller.go @@ -17,7 +17,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -300,18 +299,12 @@ func (r *ReconcileApmServer) validate(ctx context.Context, as *apmv1.ApmServer) span, vctx := apm.StartSpan(ctx, "validate", tracing.SpanTypeApp) defer span.End() - warnings, err := apmv1.Validate(as, nil) + warnings, err := validateApmServer(as, nil, r.ExposedNodeLabels) if err != nil { log.Error(err, "Validation failed") k8s.MaybeEmitErrorEvent(r.recorder, err, as, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } - if errs := commonnodelabels.ValidateAnnotation(as.Annotations, r.ExposedNodeLabels); len(errs) > 0 { - err := apierrors.NewInvalid(schema.GroupKind{Group: apmv1.GroupVersion.Group, Kind: apmv1.Kind}, as.Name, errs) - log.Error(err, "Validation failed") - k8s.MaybeEmitErrorEvent(r.recorder, err, as, events.EventReasonValidation, events.EventActionValidation, err.Error()) - return tracing.CaptureError(vctx, err) - } for _, warning := range warnings { k8s.EmitEvent(r.recorder, as, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } diff --git a/pkg/controller/apmserver/deployment.go b/pkg/controller/apmserver/deployment.go index c606a900297..f320111266a 100644 --- a/pkg/controller/apmserver/deployment.go +++ b/pkg/controller/apmserver/deployment.go @@ -15,6 +15,7 @@ import ( "k8s.io/apimachinery/pkg/types" apmv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/apm/v1" + commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/certificates" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/deployment" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" @@ -119,7 +120,7 @@ func buildConfigHash(c k8s.Client, as *apmv1.ApmServer, params PodSpecParams) (s // Changes to the downward-node-labels annotation must roll the APM Server Pods so the new annotations // are re-applied on scheduling. if as.HasDownwardNodeLabels() { - _, _ = configHash.Write([]byte(as.Annotations[apmv1.DownwardNodeLabelsAnnotation])) + _, _ = configHash.Write([]byte(as.Annotations[commonv1.DownwardNodeLabelsAnnotation])) } // - in the APMServer keystore diff --git a/pkg/controller/apmserver/webhook.go b/pkg/controller/apmserver/webhook.go new file mode 100644 index 00000000000..1b87e9b61a9 --- /dev/null +++ b/pkg/controller/apmserver/webhook.go @@ -0,0 +1,61 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package apmserver + +import ( + "context" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + apmv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/apm/v1" + commonlicense "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" + commonwebhook "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/webhook" + ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" +) + +// RegisterWebhook registers the APM Server validating webhook. +func RegisterWebhook(mgr ctrl.Manager, checker commonlicense.Checker, exposedNodeLabels commonnodelabels.NodeLabels, managedNamespaces []string) { + inner := &webhookValidator{exposedNodeLabels: exposedNodeLabels} + v := commonwebhook.NewResourceValidator[*apmv1.ApmServer](checker, managedNamespaces, inner) + wh := admission.WithValidator[*apmv1.ApmServer](mgr.GetScheme(), v) + mgr.GetWebhookServer().Register(apmv1.WebhookPath, wh) + ulog.Log.Info("Registering APM Server validating webhook", "path", apmv1.WebhookPath) +} + +type webhookValidator struct { + exposedNodeLabels commonnodelabels.NodeLabels +} + +func (v *webhookValidator) ValidateCreate(_ context.Context, obj *apmv1.ApmServer) (admission.Warnings, error) { + return validateApmServer(obj, nil, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateUpdate(_ context.Context, oldObj, newObj *apmv1.ApmServer) (admission.Warnings, error) { + return validateApmServer(newObj, oldObj, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateDelete(_ context.Context, _ *apmv1.ApmServer) (admission.Warnings, error) { + return nil, nil +} + +// validateApmServer runs the APM Server validation together with the operator's exposed-node-labels +// policy check. Both the validating webhook and the reconciler call it so the same rules are +// enforced through a single function, regardless of whether the webhook is enabled. +func validateApmServer(as *apmv1.ApmServer, old *apmv1.ApmServer, exposedNodeLabels commonnodelabels.NodeLabels) (admission.Warnings, error) { + warnings, err := apmv1.Validate(as, old) + if err != nil { + return warnings, err + } + if errs := commonnodelabels.ValidateAnnotation(as.Annotations, exposedNodeLabels); len(errs) > 0 { + return warnings, apierrors.NewInvalid( + schema.GroupKind{Group: apmv1.GroupVersion.Group, Kind: apmv1.Kind}, + as.Name, errs) + } + return warnings, nil +} diff --git a/pkg/controller/beat/common/pod.go b/pkg/controller/beat/common/pod.go index 8a3e3c99b6f..1f3755f5d3a 100644 --- a/pkg/controller/beat/common/pod.go +++ b/pkg/controller/beat/common/pod.go @@ -155,7 +155,7 @@ func buildPodTemplate( // Changes to the downward-node-labels annotation must roll the Beat Pods so the new annotations // are re-applied on scheduling. if params.Beat.HasDownwardNodeLabels() { - _, _ = configHash.Write([]byte(params.Beat.Annotations[beatv1beta1.DownwardNodeLabelsAnnotation])) + _, _ = configHash.Write([]byte(params.Beat.Annotations[commonv1.DownwardNodeLabelsAnnotation])) downwardAPIVolume := commonnodelabels.DownwardAPIVolume() waitInit, err := commonnodelabels.WaitForAnnotationsInitContainer(params.Beat.DownwardNodeLabels()) if err != nil { diff --git a/pkg/controller/beat/controller.go b/pkg/controller/beat/controller.go index 9be2ae40b39..ca6ee4e101c 100644 --- a/pkg/controller/beat/controller.go +++ b/pkg/controller/beat/controller.go @@ -11,7 +11,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -33,7 +32,6 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/events" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" - commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" @@ -192,18 +190,12 @@ func (r *ReconcileBeat) validate(ctx context.Context, beat *beatv1beta1.Beat) er span, vctx := apm.StartSpan(ctx, "validate", tracing.SpanTypeApp) defer span.End() - warnings, err := beatv1beta1.Validate(beat, nil) + warnings, err := validateBeat(beat, nil, r.ExposedNodeLabels) if err != nil { ulog.FromContext(ctx).Error(err, "Validation failed") k8s.MaybeEmitErrorEvent(r.recorder, err, beat, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } - if errs := commonnodelabels.ValidateAnnotation(beat.Annotations, r.ExposedNodeLabels); len(errs) > 0 { - err := apierrors.NewInvalid(schema.GroupKind{Group: beatv1beta1.GroupVersion.Group, Kind: beatv1beta1.Kind}, beat.Name, errs) - ulog.FromContext(ctx).Error(err, "Validation failed") - k8s.MaybeEmitErrorEvent(r.recorder, err, beat, events.EventReasonValidation, events.EventActionValidation, err.Error()) - return tracing.CaptureError(vctx, err) - } for _, warning := range warnings { k8s.EmitEvent(r.recorder, beat, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } diff --git a/pkg/controller/beat/webhook.go b/pkg/controller/beat/webhook.go new file mode 100644 index 00000000000..b786b363554 --- /dev/null +++ b/pkg/controller/beat/webhook.go @@ -0,0 +1,61 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package beat + +import ( + "context" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + beatv1beta1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/beat/v1beta1" + commonlicense "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" + commonwebhook "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/webhook" + ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" +) + +// RegisterWebhook registers the Beat validating webhook. +func RegisterWebhook(mgr ctrl.Manager, checker commonlicense.Checker, exposedNodeLabels commonnodelabels.NodeLabels, managedNamespaces []string) { + inner := &webhookValidator{exposedNodeLabels: exposedNodeLabels} + v := commonwebhook.NewResourceValidator[*beatv1beta1.Beat](checker, managedNamespaces, inner) + wh := admission.WithValidator[*beatv1beta1.Beat](mgr.GetScheme(), v) + mgr.GetWebhookServer().Register(beatv1beta1.WebhookPath, wh) + ulog.Log.Info("Registering Beat validating webhook", "path", beatv1beta1.WebhookPath) +} + +type webhookValidator struct { + exposedNodeLabels commonnodelabels.NodeLabels +} + +func (v *webhookValidator) ValidateCreate(_ context.Context, obj *beatv1beta1.Beat) (admission.Warnings, error) { + return validateBeat(obj, nil, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateUpdate(_ context.Context, oldObj, newObj *beatv1beta1.Beat) (admission.Warnings, error) { + return validateBeat(newObj, oldObj, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateDelete(_ context.Context, _ *beatv1beta1.Beat) (admission.Warnings, error) { + return nil, nil +} + +// validateBeat runs the Beat validation together with the operator's exposed-node-labels policy +// check. Both the validating webhook and the reconciler call it so the same rules are enforced +// through a single function, regardless of whether the webhook is enabled. +func validateBeat(b *beatv1beta1.Beat, old *beatv1beta1.Beat, exposedNodeLabels commonnodelabels.NodeLabels) (admission.Warnings, error) { + warnings, err := beatv1beta1.Validate(b, old) + if err != nil { + return warnings, err + } + if errs := commonnodelabels.ValidateAnnotation(b.Annotations, exposedNodeLabels); len(errs) > 0 { + return warnings, apierrors.NewInvalid( + schema.GroupKind{Group: beatv1beta1.GroupVersion.Group, Kind: beatv1beta1.Kind}, + b.Name, errs) + } + return warnings, nil +} diff --git a/pkg/controller/common/nodelabels/exposed.go b/pkg/controller/common/nodelabels/exposed.go index cac1fa40aea..71d2f5f50d3 100644 --- a/pkg/controller/common/nodelabels/exposed.go +++ b/pkg/controller/common/nodelabels/exposed.go @@ -10,7 +10,7 @@ import ( "k8s.io/apimachinery/pkg/util/validation/field" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" + commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" ) // NotAllowedNodesLabelMsg is returned when a node label requested via the downward-node-labels @@ -52,12 +52,12 @@ func (n NodeLabels) IsAllowed(nodeLabel string) bool { // when no annotation is set. func ValidateAnnotation(annotations map[string]string, exposedNodeLabels NodeLabels) field.ErrorList { var errs field.ErrorList - for _, nodeLabel := range nodelabels.FromAnnotations(annotations) { + for _, nodeLabel := range commonv1.DownwardNodeLabelsFromAnnotations(annotations) { if exposedNodeLabels.IsAllowed(nodeLabel) { continue } errs = append(errs, field.Invalid( - field.NewPath("metadata").Child("annotations", nodelabels.DownwardNodeLabelsAnnotation), + field.NewPath("metadata").Child("annotations", commonv1.DownwardNodeLabelsAnnotation), nodeLabel, NotAllowedNodesLabelMsg, )) diff --git a/pkg/controller/common/nodelabels/initcontainer.go b/pkg/controller/common/nodelabels/initcontainer.go index c180a8c72a6..a6fcdd1c193 100644 --- a/pkg/controller/common/nodelabels/initcontainer.go +++ b/pkg/controller/common/nodelabels/initcontainer.go @@ -6,14 +6,12 @@ package nodelabels import ( "bytes" - "fmt" "strings" "text/template" corev1 "k8s.io/api/core/v1" commonvolume "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/volume" - esvolume "github.com/elastic/cloud-on-k8s/v3/pkg/controller/elasticsearch/volume" ) // WaitForAnnotationsContainerName is the name of the init container that blocks Pod start @@ -27,7 +25,7 @@ expected_annotations=({{ .ExpectedAnnotations }}) annotations_file={{ .AnnotationsFile }} function annotations_exist() { for expected_annotation in "${expected_annotations[@]}"; do - if ! grep -qE "^${expected_annotation}=" "${annotations_file}"; then + if ! grep -q "^${expected_annotation}=" "${annotations_file}"; then return 1 fi done @@ -60,7 +58,7 @@ func WaitForAnnotationsInitContainer(expectedAnnotations []string) (corev1.Conta AnnotationsFile string }{ ExpectedAnnotations: strings.Join(expectedAnnotations, " "), - AnnotationsFile: fmt.Sprintf("%s/%s", esvolume.DownwardAPIMountPath, esvolume.AnnotationsFile), + AnnotationsFile: DownwardAPIVolume().AnnotationsFilePath(), }); err != nil { return corev1.Container{}, err } diff --git a/pkg/controller/common/nodelabels/initcontainer_test.go b/pkg/controller/common/nodelabels/initcontainer_test.go index 8d90a59e803..4caab33e05c 100644 --- a/pkg/controller/common/nodelabels/initcontainer_test.go +++ b/pkg/controller/common/nodelabels/initcontainer_test.go @@ -27,8 +27,9 @@ func TestWaitForAnnotationsInitContainer(t *testing.T) { script := c.Command[2] assert.Contains(t, script, "topology.kubernetes.io/zone topology.kubernetes.io/region") assert.Contains(t, script, "/mnt/elastic-internal/downward-api/annotations") - // Each expected annotation is matched at the beginning of a line followed by '=': - assert.Contains(t, script, `grep -qE "^${expected_annotation}="`) + // Each expected annotation is matched at the beginning of a line followed by '='. + // Use BRE (no -E) to match the Elasticsearch prepare-fs script. + assert.Contains(t, script, `grep -q "^${expected_annotation}="`) // No image/resources are set so they are inherited from the main container. assert.Equal(t, "", c.Image) assert.Empty(t, c.Resources.Limits) diff --git a/pkg/controller/common/nodelabels/nodelabels.go b/pkg/controller/common/nodelabels/nodelabels.go index 822d80686b3..700ad285140 100644 --- a/pkg/controller/common/nodelabels/nodelabels.go +++ b/pkg/controller/common/nodelabels/nodelabels.go @@ -24,12 +24,8 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" "github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s" ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/nodelabels" ) -// DownwardNodeLabelsAnnotation is re-exported for convenience. -const DownwardNodeLabelsAnnotation = nodelabels.DownwardNodeLabelsAnnotation - // AnnotationTarget is implemented by ECK custom resources whose managed Pods should have // Kubernetes node labels copied to their annotations via AnnotatePods. Any ECK CR that already // implements metav1.Object and GetIdentityLabels satisfies this interface once it exposes a diff --git a/pkg/controller/common/nodelabels/nodelabels_test.go b/pkg/controller/common/nodelabels/nodelabels_test.go index 4b4ecd15d1c..d72ba62e647 100644 --- a/pkg/controller/common/nodelabels/nodelabels_test.go +++ b/pkg/controller/common/nodelabels/nodelabels_test.go @@ -119,8 +119,8 @@ func TestAnnotatePods(t *testing.T) { t.Run(tt.name, func(t *testing.T) { c := k8s.NewFakeClient(tt.objects...) target := &fakeTarget{ - Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "sample"}}, - labels: tt.expectedLabels, + Pod: &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: "sample"}}, + labels: tt.expectedLabels, selector: podSelector, } results := AnnotatePods(context.Background(), c, target) diff --git a/pkg/controller/common/volume/downward_api.go b/pkg/controller/common/volume/downward_api.go index 648fb11f831..2faf1f187fa 100644 --- a/pkg/controller/common/volume/downward_api.go +++ b/pkg/controller/common/volume/downward_api.go @@ -5,6 +5,8 @@ package volume import ( + "fmt" + corev1 "k8s.io/api/core/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/elasticsearch/volume" @@ -65,3 +67,9 @@ func (d DownwardAPI) Volume() corev1.Volume { func (DownwardAPI) VolumeMount() corev1.VolumeMount { return downwardAPIVolumeMount } + +// AnnotationsFilePath returns the absolute path of the file exposing the Pod annotations within the +// downward API volume mount. It is only populated when the volume is built WithAnnotations(true). +func (DownwardAPI) AnnotationsFilePath() string { + return fmt.Sprintf("%s/%s", volume.DownwardAPIMountPath, volume.AnnotationsFile) +} diff --git a/pkg/controller/common/webhook/resource_validator.go b/pkg/controller/common/webhook/resource_validator.go index 26a51993a53..9dca2ef01da 100644 --- a/pkg/controller/common/webhook/resource_validator.go +++ b/pkg/controller/common/webhook/resource_validator.go @@ -9,46 +9,16 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/webhook/admission" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" - commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" "github.com/elastic/cloud-on-k8s/v3/pkg/utils/set" ) -// Validatable is the type constraint used by validation wrappers that need to access both the -// runtime and metadata interfaces of a resource. ECK CRDs satisfy it by embedding -// metav1.ObjectMeta and being registered with the runtime scheme. -type Validatable interface { - runtime.Object - metav1.Object -} - -// WithNodeLabelsValidation wraps a ValidateFunc with the operator's exposed-node-labels policy -// check, so that any value of the downward-node-labels annotation is validated consistently -// across resources. -func WithNodeLabelsValidation[T Validatable]( - validate ValidateFunc[T], - exposedNodeLabels commonnodelabels.NodeLabels, - groupKind schema.GroupKind, -) ValidateFunc[T] { - return func(obj T, old T) (admission.Warnings, error) { - warnings, err := validate(obj, old) - if err != nil { - return warnings, err - } - if errs := commonnodelabels.ValidateAnnotation(obj.GetAnnotations(), exposedNodeLabels); len(errs) > 0 { - return warnings, apierrors.NewInvalid(groupKind, obj.GetName(), errs) - } - return warnings, nil - } -} - // ValidateFunc is the per-resource validation callback. // obj is the object being validated, old is nil/zero on create. type ValidateFunc[T runtime.Object] func(obj T, old T) (admission.Warnings, error) diff --git a/pkg/controller/elasticsearch/driver/shared/node_labels.go b/pkg/controller/elasticsearch/driver/shared/node_labels.go deleted file mode 100644 index 5e23f9f5efd..00000000000 --- a/pkg/controller/elasticsearch/driver/shared/node_labels.go +++ /dev/null @@ -1,116 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package shared - -import ( - "context" - "encoding/json" - "fmt" - "strings" - - "go.elastic.co/apm/v2" - corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - - esv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" - "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" - "github.com/elastic/cloud-on-k8s/v3/pkg/controller/elasticsearch/sset" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s" - ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" -) - -// isPodScheduled returns if the Pod is scheduled. If true it also returns the node name. -func isPodScheduled(pod *corev1.Pod) (bool, string) { - status := pod.Status - for i := range status.Conditions { - if !(status.Conditions[i].Type == corev1.PodScheduled) { - continue - } - return status.Conditions[i].Status == corev1.ConditionTrue && pod.Spec.NodeName != "", pod.Spec.NodeName - } - return false, "" -} - -// annotatePodsWithNodeLabels annotates all the Pods with the expected node labels. -func annotatePodsWithNodeLabels(ctx context.Context, c k8s.Client, es esv1.Elasticsearch) *reconciler.Results { - span, ctx := apm.StartSpan(ctx, "annotate_pods_with_node_labels", tracing.SpanTypeApp) - defer span.End() - results := reconciler.NewResult(ctx) - if !es.HasDownwardNodeLabels() { - return results - } - actualPods, err := sset.GetActualPodsForCluster(c, es) - if err != nil { - return results.WithError(err) - } - for _, pod := range actualPods { - results.WithError(annotatePodWithNodeLabels(ctx, c, pod, es)) - } - return results -} - -func annotatePodWithNodeLabels(ctx context.Context, c k8s.Client, pod corev1.Pod, es esv1.Elasticsearch) error { - scheduled, nodeName := isPodScheduled(&pod) - if !scheduled { - return nil - } - node := &corev1.Node{} - if err := c.Get(ctx, types.NamespacedName{Name: nodeName}, node); err != nil { - return err - } - // Get the missing annotations. - podAnnotations, err := getPodAnnotations(&pod, es.DownwardNodeLabels(), node.Labels) - if err != nil { - return err - } - // Stop early if there is no annotation to set. - if len(podAnnotations) == 0 { - return nil - } - ulog.FromContext(ctx).Info("Setting Pod annotations from node labels", "err", err, "namespace", es.Namespace, "es_name", es.Name, "pod", pod.Name, "annotations", podAnnotations) - mergePatch, err := json.Marshal(map[string]any{ - "metadata": map[string]any{ - "annotations": podAnnotations, - }, - }) - if err != nil { - return err - } - if err := c.Patch(ctx, &pod, client.RawPatch(types.StrategicMergePatchType, mergePatch)); err != nil && !errors.IsNotFound(err) { - return err - } - return nil -} - -// getPodAnnotations returns missing annotations, and their values, expected on a given Pod. -// It also ensures that labels exist on the K8S node, if not the case an error is returned. -func getPodAnnotations(pod *corev1.Pod, expectedAnnotations []string, nodeLabels map[string]string) (map[string]string, error) { - podAnnotations := make(map[string]string) - var missingLabels []string - for _, expectedAnnotation := range expectedAnnotations { - value, ok := nodeLabels[expectedAnnotation] - if !ok { - missingLabels = append(missingLabels, expectedAnnotation) - continue - } - // Check if the annotations is already set - if _, alreadyExists := pod.Annotations[expectedAnnotation]; alreadyExists { - continue - } - podAnnotations[expectedAnnotation] = value - } - if len(missingLabels) > 0 { - return nil, - fmt.Errorf( - "following annotations are expected to be set on Pod %s/%s but do not exist as node labels: %s", - pod.Namespace, - pod.Name, - strings.Join(missingLabels, ",")) - } - return podAnnotations, nil -} diff --git a/pkg/controller/elasticsearch/driver/shared/node_labels_test.go b/pkg/controller/elasticsearch/driver/shared/node_labels_test.go deleted file mode 100644 index 607e061d404..00000000000 --- a/pkg/controller/elasticsearch/driver/shared/node_labels_test.go +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one -// or more contributor license agreements. Licensed under the Elastic License 2.0; -// you may not use this file except in compliance with the Elastic License 2.0. - -package shared - -import ( - "context" - "testing" - - "github.com/stretchr/testify/assert" - corev1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "sigs.k8s.io/controller-runtime/pkg/client" - - esv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/elasticsearch/v1" - "github.com/elastic/cloud-on-k8s/v3/pkg/utils/k8s" -) - -// expectedPodsAnnotations holds pod -> expectedAnnotation -> expectedValue -type expectedPodsAnnotations map[string]map[string]string - -func (e expectedPodsAnnotations) isEmpty() bool { - return len(e) == 0 -} - -func (e expectedPodsAnnotations) assertPodsMatch(t *testing.T, pods []corev1.Pod) { - t.Helper() - assert.Equal(t, len(e), len(pods), "expected %d Pods with annotations, got %d Pods", len(e), len(pods)) - for _, pod := range pods { - expectedAnnotations, exists := e[pod.Name] - assert.True(t, exists) - for expectedAnnotation, expectedValue := range expectedAnnotations { - actualValue, exists := pod.Annotations[expectedAnnotation] - assert.True(t, exists, "Pod %s: expected annotation %s", pod.Name, expectedAnnotation) - assert.Equal( - t, - expectedValue, - actualValue, - "Pod %s: expected value \"%s\" for annotation \"%s\", got value \"%s\"", - pod.Name, expectedValue, expectedAnnotation, actualValue, - ) - } - } -} - -const esName = "elasticsearch-sample" - -func Test_annotatePodsWithNodeLabels(t *testing.T) { - type args struct { - ctx context.Context - es *esv1.Elasticsearch - objects []client.Object - } - tests := []struct { - name string - args args - wantErrMsg string - expectedAnnotations expectedPodsAnnotations - }{ - { - name: "No annotations on K8S nodes", - args: args{ - es: &esv1.Elasticsearch{ - ObjectMeta: metav1.ObjectMeta{ - Name: esName, - Namespace: "ns", - Annotations: map[string]string{"eck.k8s.elastic.co/downward-node-labels": "topology.kubernetes.io/region,topology.kubernetes.io/zone"}, - }, - }, - objects: []client.Object{ - newPodBuilder("elasticsearch-sample-es-default-0").scheduledOn("k8s-node-0").build(), - newPodBuilder("elasticsearch-sample-es-default-1").scheduledOn("k8s-node-1").build(), - &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "k8s-node-0"}}, - &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "k8s-node-1"}}, - }, - ctx: context.Background(), - }, - wantErrMsg: "following annotations are expected to be set on Pod ns/elasticsearch-sample-es-default-0 but do not exist as node labels: topology.kubernetes.io/region,topology.kubernetes.io/zone", - }, - { - name: "No initial annotations on the Pods", - args: args{ - es: &esv1.Elasticsearch{ - ObjectMeta: metav1.ObjectMeta{ - Name: esName, - Namespace: "ns", - Annotations: map[string]string{"eck.k8s.elastic.co/downward-node-labels": "topology.kubernetes.io/region,topology.kubernetes.io/zone"}, - }, - }, - objects: []client.Object{ - newPodBuilder("elasticsearch-sample-es-default-0").scheduledOn("k8s-node-0").build(), - newPodBuilder("elasticsearch-sample-es-default-1").scheduledOn("k8s-node-1").build(), - &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "k8s-node-0", Labels: map[string]string{"topology.kubernetes.io/region": "europe-west1", "topology.kubernetes.io/zone": "europe-west1-a"}}}, - &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "k8s-node-1", Labels: map[string]string{"topology.kubernetes.io/region": "europe-west1", "topology.kubernetes.io/zone": "europe-west1-b"}}}, - }, - ctx: context.Background(), - }, - expectedAnnotations: expectedPodsAnnotations{ - "elasticsearch-sample-es-default-0": { - "topology.kubernetes.io/region": "europe-west1", - "topology.kubernetes.io/zone": "europe-west1-a", - }, - "elasticsearch-sample-es-default-1": { - "topology.kubernetes.io/region": "europe-west1", - "topology.kubernetes.io/zone": "europe-west1-b", - }, - }, - }, - { - name: "With initial annotations on the Pods", - args: args{ - es: &esv1.Elasticsearch{ - ObjectMeta: metav1.ObjectMeta{ - Name: esName, - Namespace: "ns", - Annotations: map[string]string{"eck.k8s.elastic.co/downward-node-labels": "topology.kubernetes.io/region,topology.kubernetes.io/zone"}, - }, - }, - objects: []client.Object{ - newPodBuilder("elasticsearch-sample-es-default-0").scheduledOn("k8s-node-0").withAnnotation(map[string]string{"foo": "bar"}).build(), - newPodBuilder("elasticsearch-sample-es-default-1").scheduledOn("k8s-node-1").withAnnotation(map[string]string{"foo": "bar"}).build(), - &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "k8s-node-0", Labels: map[string]string{"topology.kubernetes.io/region": "europe-west1", "topology.kubernetes.io/zone": "europe-west1-a"}}}, - &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "k8s-node-1", Labels: map[string]string{"topology.kubernetes.io/region": "europe-west1", "topology.kubernetes.io/zone": "europe-west1-b"}}}, - }, - ctx: context.Background(), - }, - expectedAnnotations: expectedPodsAnnotations{ - "elasticsearch-sample-es-default-0": { - "topology.kubernetes.io/region": "europe-west1", - "topology.kubernetes.io/zone": "europe-west1-a", - "foo": "bar", - }, - "elasticsearch-sample-es-default-1": { - "topology.kubernetes.io/region": "europe-west1", - "topology.kubernetes.io/zone": "europe-west1-b", - "foo": "bar", - }, - }, - }, - { - name: "Retain existing annotations", - args: args{ - es: &esv1.Elasticsearch{ - ObjectMeta: metav1.ObjectMeta{ - Name: esName, - Namespace: "ns", - Annotations: map[string]string{"eck.k8s.elastic.co/downward-node-labels": "topology.kubernetes.io/region,topology.kubernetes.io/zone"}, - }, - }, - objects: []client.Object{ - newPodBuilder("elasticsearch-sample-es-default-0").scheduledOn("k8s-node-0").withAnnotation(map[string]string{"foo": "bar", "topology.kubernetes.io/region": "existing-annotation"}).build(), - newPodBuilder("elasticsearch-sample-es-default-1").scheduledOn("k8s-node-1").withAnnotation(map[string]string{"foo": "bar", "topology.kubernetes.io/region": "existing-annotation"}).build(), - &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "k8s-node-0", Labels: map[string]string{"topology.kubernetes.io/region": "europe-west1", "topology.kubernetes.io/zone": "europe-west1-a"}}}, - &corev1.Node{ObjectMeta: metav1.ObjectMeta{Name: "k8s-node-1", Labels: map[string]string{"topology.kubernetes.io/region": "europe-west1", "topology.kubernetes.io/zone": "europe-west1-b"}}}, - }, - ctx: context.Background(), - }, - expectedAnnotations: expectedPodsAnnotations{ - "elasticsearch-sample-es-default-0": { - "topology.kubernetes.io/region": "existing-annotation", - "topology.kubernetes.io/zone": "europe-west1-a", - "foo": "bar", - }, - "elasticsearch-sample-es-default-1": { - "topology.kubernetes.io/region": "existing-annotation", - "topology.kubernetes.io/zone": "europe-west1-b", - "foo": "bar", - }, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - k8sClient := k8s.NewFakeClient(append(tt.args.objects, tt.args.es)...) - got := annotatePodsWithNodeLabels(tt.args.ctx, k8sClient, *tt.args.es) - _, err := got.Aggregate() - if tt.wantErrMsg != "" { - assert.Containsf(t, err.Error(), tt.wantErrMsg, "expected error containing %q, got %s", tt.wantErrMsg, err) - } else { - assert.NoError(t, err) - } - if !tt.expectedAnnotations.isEmpty() { - podList := &corev1.PodList{} - assert.NoError(t, k8sClient.List(context.Background(), podList)) - tt.expectedAnnotations.assertPodsMatch(t, podList.Items) - } - }) - } -} - -type podBuilder struct { - podName, nodeName string - scheduled bool - annotations map[string]string -} - -func newPodBuilder(podName string) *podBuilder { - return &podBuilder{ - podName: podName, - } -} - -func (pb *podBuilder) scheduledOn(nodeName string) *podBuilder { - pb.scheduled = true - pb.nodeName = nodeName - return pb -} - -func (pb *podBuilder) withAnnotation(annotations map[string]string) *podBuilder { - pb.annotations = annotations - return pb -} - -func (pb *podBuilder) build() *corev1.Pod { - pod := corev1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: pb.podName, Namespace: "ns", - Labels: map[string]string{"elasticsearch.k8s.elastic.co/cluster-name": esName}, - Annotations: pb.annotations, - }, - Spec: corev1.PodSpec{ - NodeName: pb.nodeName, - }, - } - if pb.scheduled { - pod.Status = corev1.PodStatus{ - Phase: "", - Conditions: []corev1.PodCondition{ - { - Type: corev1.PodScheduled, - Status: corev1.ConditionTrue, - }, - }, - } - } - return &pod -} diff --git a/pkg/controller/elasticsearch/driver/shared/reconcile.go b/pkg/controller/elasticsearch/driver/shared/reconcile.go index 4da89be0a23..c21dd9f2645 100644 --- a/pkg/controller/elasticsearch/driver/shared/reconcile.go +++ b/pkg/controller/elasticsearch/driver/shared/reconcile.go @@ -25,6 +25,7 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/events" commonlicense "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/metadata" + "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/version" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/elasticsearch/bootstrap" @@ -214,7 +215,7 @@ func ReconcileSharedResources( // Patch the Pods to add the expected node labels as annotations. Record the error, if any, but do not stop the // reconciliation loop as we don't want to prevent other updates from being applied to the cluster. - results.WithResults(annotatePodsWithNodeLabels(ctx, client, es)) + results.WithResults(nodelabels.AnnotatePods(ctx, client, &es)) // Verify the operator supports existing pods if err := VerifySupportsExistingPods(resourcesState.CurrentPods, params.SupportedVersions); err != nil { diff --git a/pkg/controller/kibana/controller.go b/pkg/controller/kibana/controller.go index 491992669f4..651f08c5b06 100644 --- a/pkg/controller/kibana/controller.go +++ b/pkg/controller/kibana/controller.go @@ -14,7 +14,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -29,7 +28,6 @@ import ( "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/events" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/finalizer" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/keystore" - commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/operator" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/reconciler" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/tracing" @@ -213,18 +211,12 @@ func (r *ReconcileKibana) validate(ctx context.Context, kb *kbv1.Kibana) error { span, vctx := apm.StartSpan(ctx, "validate", tracing.SpanTypeApp) defer span.End() - warnings, err := kbv1.Validate(kb, nil) + warnings, err := validateKibana(kb, nil, r.params.ExposedNodeLabels) if err != nil { ulog.FromContext(ctx).Error(err, "Validation failed") k8s.MaybeEmitErrorEvent(r.recorder, err, kb, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } - if errs := commonnodelabels.ValidateAnnotation(kb.Annotations, r.params.ExposedNodeLabels); len(errs) > 0 { - err := apierrors.NewInvalid(schema.GroupKind{Group: kbv1.GroupVersion.Group, Kind: kbv1.Kind}, kb.Name, errs) - ulog.FromContext(ctx).Error(err, "Validation failed") - k8s.MaybeEmitErrorEvent(r.recorder, err, kb, events.EventReasonValidation, events.EventActionValidation, err.Error()) - return tracing.CaptureError(vctx, err) - } for _, warning := range warnings { k8s.EmitEvent(r.recorder, kb, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } diff --git a/pkg/controller/kibana/driver.go b/pkg/controller/kibana/driver.go index e0841313f08..8bc973544bd 100644 --- a/pkg/controller/kibana/driver.go +++ b/pkg/controller/kibana/driver.go @@ -17,6 +17,7 @@ import ( toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/client" + commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" kbv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/kibana/v1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/association" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common" @@ -286,7 +287,7 @@ func (d *driver) deploymentParams(ctx context.Context, kb *kbv1.Kibana, policyAn // Changes to the downward-node-labels annotation must roll the Kibana Pods so the new annotations // are re-applied on scheduling. if kb.HasDownwardNodeLabels() { - _, _ = configHash.Write([]byte(kb.Annotations[kbv1.DownwardNodeLabelsAnnotation])) + _, _ = configHash.Write([]byte(kb.Annotations[commonv1.DownwardNodeLabelsAnnotation])) } if kb.Spec.HTTP.TLS.Enabled() { diff --git a/pkg/controller/kibana/webhook.go b/pkg/controller/kibana/webhook.go new file mode 100644 index 00000000000..00df1165fdd --- /dev/null +++ b/pkg/controller/kibana/webhook.go @@ -0,0 +1,61 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package kibana + +import ( + "context" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + kbv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/kibana/v1" + commonlicense "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" + commonwebhook "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/webhook" + ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" +) + +// RegisterWebhook registers the Kibana validating webhook. +func RegisterWebhook(mgr ctrl.Manager, checker commonlicense.Checker, exposedNodeLabels commonnodelabels.NodeLabels, managedNamespaces []string) { + inner := &webhookValidator{exposedNodeLabels: exposedNodeLabels} + v := commonwebhook.NewResourceValidator[*kbv1.Kibana](checker, managedNamespaces, inner) + wh := admission.WithValidator[*kbv1.Kibana](mgr.GetScheme(), v) + mgr.GetWebhookServer().Register(kbv1.WebhookPath, wh) + ulog.Log.Info("Registering Kibana validating webhook", "path", kbv1.WebhookPath) +} + +type webhookValidator struct { + exposedNodeLabels commonnodelabels.NodeLabels +} + +func (v *webhookValidator) ValidateCreate(_ context.Context, obj *kbv1.Kibana) (admission.Warnings, error) { + return validateKibana(obj, nil, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateUpdate(_ context.Context, oldObj, newObj *kbv1.Kibana) (admission.Warnings, error) { + return validateKibana(newObj, oldObj, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateDelete(_ context.Context, _ *kbv1.Kibana) (admission.Warnings, error) { + return nil, nil +} + +// validateKibana runs the Kibana validation together with the operator's exposed-node-labels policy +// check. Both the validating webhook and the reconciler call it so the same rules are enforced +// through a single function, regardless of whether the webhook is enabled. +func validateKibana(k *kbv1.Kibana, old *kbv1.Kibana, exposedNodeLabels commonnodelabels.NodeLabels) (admission.Warnings, error) { + warnings, err := kbv1.Validate(k, old) + if err != nil { + return warnings, err + } + if errs := commonnodelabels.ValidateAnnotation(k.Annotations, exposedNodeLabels); len(errs) > 0 { + return warnings, apierrors.NewInvalid( + schema.GroupKind{Group: kbv1.GroupVersion.Group, Kind: kbv1.Kind}, + k.Name, errs) + } + return warnings, nil +} diff --git a/pkg/controller/logstash/pod.go b/pkg/controller/logstash/pod.go index ecc01f4fe41..9fd86432d08 100644 --- a/pkg/controller/logstash/pod.go +++ b/pkg/controller/logstash/pod.go @@ -92,7 +92,7 @@ func buildPodTemplate(params Params, configHash hash.Hash32) (corev1.PodTemplate // Changes to the downward-node-labels annotation must roll the Logstash Pods so the new annotations // are re-applied on scheduling. if params.Logstash.HasDownwardNodeLabels() { - _, _ = configHash.Write([]byte(params.Logstash.Annotations[logstashv1alpha1.DownwardNodeLabelsAnnotation])) + _, _ = configHash.Write([]byte(params.Logstash.Annotations[commonv1.DownwardNodeLabelsAnnotation])) } annotations := map[string]string{ diff --git a/pkg/controller/maps/controller.go b/pkg/controller/maps/controller.go index ec41cce5a0c..e6a72d53ef9 100644 --- a/pkg/controller/maps/controller.go +++ b/pkg/controller/maps/controller.go @@ -16,7 +16,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -25,6 +24,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" emsv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/maps/v1alpha1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/association" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common" @@ -295,18 +295,12 @@ func (r *ReconcileMapsServer) validate(ctx context.Context, ems emsv1alpha1.Elas span, vctx := apm.StartSpan(ctx, "validate", tracing.SpanTypeApp) defer span.End() - warnings, err := emsv1alpha1.Validate(&ems, nil) + warnings, err := validateMapsServer(&ems, nil, r.ExposedNodeLabels) if err != nil { ulog.FromContext(ctx).Error(err, "Validation failed") k8s.MaybeEmitErrorEvent(r.recorder, err, &ems, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } - if errs := commonnodelabels.ValidateAnnotation(ems.Annotations, r.ExposedNodeLabels); len(errs) > 0 { - err := apierrors.NewInvalid(schema.GroupKind{Group: emsv1alpha1.GroupVersion.Group, Kind: emsv1alpha1.Kind}, ems.Name, errs) - ulog.FromContext(ctx).Error(err, "Validation failed") - k8s.MaybeEmitErrorEvent(r.recorder, err, &ems, events.EventReasonValidation, events.EventActionValidation, err.Error()) - return tracing.CaptureError(vctx, err) - } for _, warning := range warnings { k8s.EmitEvent(r.recorder, &ems, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } @@ -345,7 +339,7 @@ func buildConfigHash(c k8s.Client, ems emsv1alpha1.ElasticMapsServer, configSecr // Changes to the downward-node-labels annotation must roll the Elastic Maps Server Pods so the new // annotations are re-applied on scheduling. if ems.HasDownwardNodeLabels() { - _, _ = configHash.Write([]byte(ems.Annotations[emsv1alpha1.DownwardNodeLabelsAnnotation])) + _, _ = configHash.Write([]byte(ems.Annotations[commonv1.DownwardNodeLabelsAnnotation])) } // - in the Elastic Maps Server TLS certificates diff --git a/pkg/controller/maps/webhook.go b/pkg/controller/maps/webhook.go new file mode 100644 index 00000000000..d7588172a71 --- /dev/null +++ b/pkg/controller/maps/webhook.go @@ -0,0 +1,61 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package maps + +import ( + "context" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + emsv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/maps/v1alpha1" + commonlicense "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" + commonwebhook "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/webhook" + ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" +) + +// RegisterWebhook registers the Elastic Maps Server validating webhook. +func RegisterWebhook(mgr ctrl.Manager, checker commonlicense.Checker, exposedNodeLabels commonnodelabels.NodeLabels, managedNamespaces []string) { + inner := &webhookValidator{exposedNodeLabels: exposedNodeLabels} + v := commonwebhook.NewResourceValidator[*emsv1alpha1.ElasticMapsServer](checker, managedNamespaces, inner) + wh := admission.WithValidator[*emsv1alpha1.ElasticMapsServer](mgr.GetScheme(), v) + mgr.GetWebhookServer().Register(emsv1alpha1.WebhookPath, wh) + ulog.Log.Info("Registering Elastic Maps Server validating webhook", "path", emsv1alpha1.WebhookPath) +} + +type webhookValidator struct { + exposedNodeLabels commonnodelabels.NodeLabels +} + +func (v *webhookValidator) ValidateCreate(_ context.Context, obj *emsv1alpha1.ElasticMapsServer) (admission.Warnings, error) { + return validateMapsServer(obj, nil, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateUpdate(_ context.Context, oldObj, newObj *emsv1alpha1.ElasticMapsServer) (admission.Warnings, error) { + return validateMapsServer(newObj, oldObj, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateDelete(_ context.Context, _ *emsv1alpha1.ElasticMapsServer) (admission.Warnings, error) { + return nil, nil +} + +// validateMapsServer runs the Elastic Maps Server validation together with the operator's +// exposed-node-labels policy check. Both the validating webhook and the reconciler call it so the +// same rules are enforced through a single function, regardless of whether the webhook is enabled. +func validateMapsServer(m *emsv1alpha1.ElasticMapsServer, old *emsv1alpha1.ElasticMapsServer, exposedNodeLabels commonnodelabels.NodeLabels) (admission.Warnings, error) { + warnings, err := emsv1alpha1.Validate(m, old) + if err != nil { + return warnings, err + } + if errs := commonnodelabels.ValidateAnnotation(m.Annotations, exposedNodeLabels); len(errs) > 0 { + return warnings, apierrors.NewInvalid( + schema.GroupKind{Group: emsv1alpha1.GroupVersion.Group, Kind: emsv1alpha1.Kind}, + m.Name, errs) + } + return warnings, nil +} diff --git a/pkg/controller/packageregistry/controller.go b/pkg/controller/packageregistry/controller.go index 056ca1e1ed4..27ff9d157f2 100644 --- a/pkg/controller/packageregistry/controller.go +++ b/pkg/controller/packageregistry/controller.go @@ -15,7 +15,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" toolsevents "k8s.io/client-go/tools/events" "sigs.k8s.io/controller-runtime/pkg/controller" @@ -24,6 +23,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + commonv1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/common/v1" eprv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/packageregistry/v1alpha1" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common" "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/certificates" @@ -263,18 +263,12 @@ func (r *ReconcilePackageRegistry) validate(ctx context.Context, epr eprv1alpha1 span, vctx := apm.StartSpan(ctx, "validate", tracing.SpanTypeApp) defer span.End() - warnings, err := eprv1alpha1.Validate(&epr, nil) + warnings, err := validatePackageRegistry(&epr, nil, r.ExposedNodeLabels) if err != nil { ulog.FromContext(ctx).Error(err, "Validation failed") k8s.MaybeEmitErrorEvent(r.recorder, err, &epr, events.EventReasonValidation, events.EventActionValidation, err.Error()) return tracing.CaptureError(vctx, err) } - if errs := commonnodelabels.ValidateAnnotation(epr.Annotations, r.ExposedNodeLabels); len(errs) > 0 { - err := apierrors.NewInvalid(schema.GroupKind{Group: eprv1alpha1.GroupVersion.Group, Kind: eprv1alpha1.Kind}, epr.Name, errs) - ulog.FromContext(ctx).Error(err, "Validation failed") - k8s.MaybeEmitErrorEvent(r.recorder, err, &epr, events.EventReasonValidation, events.EventActionValidation, err.Error()) - return tracing.CaptureError(vctx, err) - } for _, warning := range warnings { k8s.EmitEvent(r.recorder, &epr, corev1.EventTypeWarning, events.EventReasonValidation, events.EventActionValidation, warning) } @@ -320,7 +314,7 @@ func buildConfigHash(epr eprv1alpha1.PackageRegistry, configSecret corev1.Secret // Changes to the downward-node-labels annotation must roll the Package Registry Pods so the new // annotations are re-applied on scheduling. if epr.HasDownwardNodeLabels() { - _, _ = configHash.Write([]byte(epr.Annotations[eprv1alpha1.DownwardNodeLabelsAnnotation])) + _, _ = configHash.Write([]byte(epr.Annotations[commonv1.DownwardNodeLabelsAnnotation])) } return fmt.Sprint(configHash.Sum32()) diff --git a/pkg/controller/packageregistry/webhook.go b/pkg/controller/packageregistry/webhook.go new file mode 100644 index 00000000000..d1bb5d73406 --- /dev/null +++ b/pkg/controller/packageregistry/webhook.go @@ -0,0 +1,61 @@ +// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one +// or more contributor license agreements. Licensed under the Elastic License 2.0; +// you may not use this file except in compliance with the Elastic License 2.0. + +package packageregistry + +import ( + "context" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime/schema" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/webhook/admission" + + eprv1alpha1 "github.com/elastic/cloud-on-k8s/v3/pkg/apis/packageregistry/v1alpha1" + commonlicense "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/license" + commonnodelabels "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/nodelabels" + commonwebhook "github.com/elastic/cloud-on-k8s/v3/pkg/controller/common/webhook" + ulog "github.com/elastic/cloud-on-k8s/v3/pkg/utils/log" +) + +// RegisterWebhook registers the Elastic Package Registry validating webhook. +func RegisterWebhook(mgr ctrl.Manager, checker commonlicense.Checker, exposedNodeLabels commonnodelabels.NodeLabels, managedNamespaces []string) { + inner := &webhookValidator{exposedNodeLabels: exposedNodeLabels} + v := commonwebhook.NewResourceValidator[*eprv1alpha1.PackageRegistry](checker, managedNamespaces, inner) + wh := admission.WithValidator[*eprv1alpha1.PackageRegistry](mgr.GetScheme(), v) + mgr.GetWebhookServer().Register(eprv1alpha1.WebhookPath, wh) + ulog.Log.Info("Registering Elastic Package Registry validating webhook", "path", eprv1alpha1.WebhookPath) +} + +type webhookValidator struct { + exposedNodeLabels commonnodelabels.NodeLabels +} + +func (v *webhookValidator) ValidateCreate(_ context.Context, obj *eprv1alpha1.PackageRegistry) (admission.Warnings, error) { + return validatePackageRegistry(obj, nil, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateUpdate(_ context.Context, oldObj, newObj *eprv1alpha1.PackageRegistry) (admission.Warnings, error) { + return validatePackageRegistry(newObj, oldObj, v.exposedNodeLabels) +} + +func (v *webhookValidator) ValidateDelete(_ context.Context, _ *eprv1alpha1.PackageRegistry) (admission.Warnings, error) { + return nil, nil +} + +// validatePackageRegistry runs the Elastic Package Registry validation together with the operator's +// exposed-node-labels policy check. Both the validating webhook and the reconciler call it so the +// same rules are enforced through a single function, regardless of whether the webhook is enabled. +func validatePackageRegistry(epr *eprv1alpha1.PackageRegistry, old *eprv1alpha1.PackageRegistry, exposedNodeLabels commonnodelabels.NodeLabels) (admission.Warnings, error) { + warnings, err := eprv1alpha1.Validate(epr, old) + if err != nil { + return warnings, err + } + if errs := commonnodelabels.ValidateAnnotation(epr.Annotations, exposedNodeLabels); len(errs) > 0 { + return warnings, apierrors.NewInvalid( + schema.GroupKind{Group: eprv1alpha1.GroupVersion.Group, Kind: eprv1alpha1.Kind}, + epr.Name, errs) + } + return warnings, nil +}