diff --git a/README.md b/README.md index cf8e458..fd44daf 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ spec: - [x] Local file / HTTP discovery engine #3 - [x] Hostname / node name discovery engine #4 - [x] Network config discovery engine #5 -- [ ] Taint management #6 +- [x] Taint management #6 - [ ] Prometheus Exporter #7 - [ ] Better RBAC / admission webhook #8 @@ -114,6 +114,36 @@ The [Sprig library](http://masterminds.github.io/sprig/) is available for advanc > > Example: "@@@foo+bar.foobar----." will be rendered as "foo_bar.foobar". +### Taint Templates + +The `taintTemplates` section defines which Kubernetes taints Topomatik will manage. The map is keyed by the taint key; each entry has a `value` and an `effect`, both Go templates (same engine and Sprig functions as label templates). + +```yaml +taintTemplates: + topology.plaffitt.kubernetes.io/specialized: + value: "{{ .hostname.zone }}" + effect: NoSchedule +``` + +Allowed rendered effects: `NoSchedule`, `PreferNoSchedule`, `NoExecute`. An effect that renders to an unsupported value preserves the existing taint on the node and logs a warning. + +Rendered values are sanitized using the same rules as label values (see the note above). + +If `effect` renders to an empty string, the taint is not applied; if it already exists on the node it will be removed. This lets you conditionally manage a taint, e.g.: + +```yaml +taintTemplates: + topology.plaffitt.kubernetes.io/specialized: + value: "{{ .hostname.zone }}" + effect: '{{ if eq .hostname.zone "eu-west" }}NoSchedule{{ end }}' +``` + +> [!NOTE] +> Conditional management of labels is driven by an empty rendered `value`, but for taints it is driven by an empty rendered `effect`. A taint with an empty `value` is still a valid Kubernetes taint, whereas an empty `effect` is the natural signal that the taint should not exist. + +> [!NOTE] +> Topomatik only manages taints whose key appears in `taintTemplates`. Taints set by other controllers or by the user (e.g. `kubectl taint`) on different keys are preserved on every reconciliation. Removing an entry from `taintTemplates` does not remove the corresponding taint from the node; remove it manually with `kubectl taint nodes -` or render the `effect` to an empty string. + ### Auto-Discovery Engine Configuration Each auto-discovery engine has its own configuration section. diff --git a/cmd/main.go b/cmd/main.go index 8c0f583..d92a654 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -69,7 +69,7 @@ func main() { scheduler := schedulers.NewSometimesWithDebounceChannel(config.MinimumReconciliationInterval) - ctrl, err := controller.New(k8sClientset, scheduler, config.LabelTemplates) + ctrl, err := controller.New(k8sClientset, scheduler, config.LabelTemplates, config.TaintTemplates) if err != nil { panic(err) } diff --git a/internal/config/config.go b/internal/config/config.go index 276c679..d5e6749 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -17,8 +17,9 @@ import ( ) type Config struct { - LabelTemplates map[string]string `yaml:"labelTemplates"` - MinimumReconciliationInterval time.Duration `yaml:"minimumReconciliationInterval"` + LabelTemplates map[string]string `yaml:"labelTemplates"` + TaintTemplates map[string]TaintTemplate `yaml:"taintTemplates"` + MinimumReconciliationInterval time.Duration `yaml:"minimumReconciliationInterval"` LLDP EngineConfig[lldp.Config] `yaml:"lldp"` Files files.Config `yaml:"files" validate:"dive"` @@ -27,6 +28,11 @@ type Config struct { Network EngineConfig[network.Config] `yaml:"network"` } +type TaintTemplate struct { + Value string `yaml:"value"` + Effect string `yaml:"effect"` +} + type EngineConfig[T any] struct { Config T `yaml:",inline"` Enabled bool `yaml:"enabled"` diff --git a/internal/controller/controller.go b/internal/controller/controller.go index f65112a..96fa581 100644 --- a/internal/controller/controller.go +++ b/internal/controller/controller.go @@ -13,6 +13,7 @@ import ( "github.com/Masterminds/sprig" "github.com/enix/topomatik/internal/autodiscovery" + "github.com/enix/topomatik/internal/config" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -27,20 +28,27 @@ type ReconciliationScheduler interface { C() <-chan struct{} } +type taintTemplate struct { + value *template.Template + effect *template.Template +} + type Controller struct { nodeName string clientset *kubernetes.Clientset labelTemplates map[string]*template.Template + taintTemplates map[string]*taintTemplate engines []*autodiscovery.Engine discoveryData map[string]map[string]string scheduler ReconciliationScheduler } -func New(clientset *kubernetes.Clientset, scheduler ReconciliationScheduler, labelTemplates map[string]string) (*Controller, error) { +func New(clientset *kubernetes.Clientset, scheduler ReconciliationScheduler, labelTemplates map[string]string, taintTemplates map[string]config.TaintTemplate) (*Controller, error) { controller := &Controller{ nodeName: os.Getenv("NODE_NAME"), clientset: clientset, labelTemplates: map[string]*template.Template{}, + taintTemplates: map[string]*taintTemplate{}, engines: []*autodiscovery.Engine{}, discoveryData: map[string]map[string]string{}, scheduler: scheduler, @@ -53,6 +61,21 @@ func New(clientset *kubernetes.Clientset, scheduler ReconciliationScheduler, lab } } + for key, t := range taintTemplates { + value := template.New("taint:" + key + ":value").Funcs(sprig.FuncMap()).Option("missingkey=error") + if _, err := value.Parse(t.Value); err != nil { + return nil, err + } + effect := template.New("taint:" + key + ":effect").Funcs(sprig.FuncMap()).Option("missingkey=error") + if _, err := effect.Parse(t.Effect); err != nil { + return nil, err + } + controller.taintTemplates[key] = &taintTemplate{ + value: value, + effect: effect, + } + } + return controller, nil } @@ -119,6 +142,13 @@ func (c *Controller) watchNode() error { return nil } +func sanitizeLabelValue(value string) string { + sanitized := labelRegexp.ReplaceAllString(value, "_") + return strings.TrimFunc(sanitized, func(r rune) bool { + return strings.Contains("_.-", string(r)) + }) +} + func (c *Controller) reconcileNode() error { slog.Debug("reconciling node") node, err := c.clientset.CoreV1().Nodes().Get(context.Background(), c.nodeName, metav1.GetOptions{}) @@ -127,18 +157,39 @@ func (c *Controller) reconcileNode() error { return nil } + labels := c.computeLabelPatch(node) + upsertTaints, deleteTaintKeys := c.computeTaintOps(node) + + if len(labels) == 0 && len(upsertTaints) == 0 && len(deleteTaintKeys) == 0 { + slog.Debug("no changes detected") + return nil + } + + patchBytes, err := buildNodeStrategicMergePatch(labels, upsertTaints, deleteTaintKeys) + if err != nil { + return fmt.Errorf("failed to marshal patch: %w", err) + } + + _, err = c.clientset.CoreV1().Nodes().Patch(context.Background(), node.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{}) + if err != nil { + return fmt.Errorf("could not patch node %s: %w", c.nodeName, err) + } + + slog.Info("node patched", "labels", len(labels), "taintsUpserted", len(upsertTaints), "taintsDeleted", len(deleteTaintKeys)) + + return nil +} + +func (c *Controller) computeLabelPatch(node *corev1.Node) map[string]any { labels := map[string]any{} + for label, tmpl := range c.labelTemplates { value := &bytes.Buffer{} - err := tmpl.Execute(value, c.discoveryData) - if err != nil { + if err := tmpl.Execute(value, c.discoveryData); err != nil { slog.Warn("could not render template", "label", label, "error", err) continue } - sanitizedValue := labelRegexp.ReplaceAllString(value.String(), "_") - sanitizedValue = strings.TrimFunc(sanitizedValue, func(r rune) bool { - return strings.Contains("_.-", string(r)) - }) + sanitizedValue := sanitizeLabelValue(value.String()) currentValue, exists := node.Labels[label] switch { @@ -151,27 +202,84 @@ func (c *Controller) reconcileNode() error { } } - if len(labels) == 0 { - slog.Debug("no label changes detected") - return nil + return labels +} + +func buildNodeStrategicMergePatch(labels map[string]any, upsertTaints []corev1.Taint, deleteTaintKeys []string) ([]byte, error) { + patch := map[string]any{} + if len(labels) > 0 { + patch["metadata"] = map[string]any{"labels": labels} } - patch := map[string]any{ - "metadata": map[string]any{ - "labels": labels, - }, + if len(upsertTaints) > 0 || len(deleteTaintKeys) > 0 { + taints := make([]any, 0, len(upsertTaints)+len(deleteTaintKeys)) + for _, t := range upsertTaints { + taints = append(taints, t) + } + + for _, key := range deleteTaintKeys { + taints = append(taints, map[string]any{"key": key, "$patch": "delete"}) + } + + patch["spec"] = map[string]any{"taints": taints} } - patchBytes, err := json.Marshal(patch) - if err != nil { - return fmt.Errorf("failed to marshal patch: %w", err) + + return json.Marshal(patch) +} + +func (c *Controller) computeTaintOps(node *corev1.Node) (upsert []corev1.Taint, deleteKeys []string) { + if len(c.taintTemplates) == 0 { + return nil, nil } - _, err = c.clientset.CoreV1().Nodes().Patch(context.Background(), node.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{}) - if err != nil { - return fmt.Errorf("could not patch node %s: %w", c.nodeName, err) + currentByKey := map[string]corev1.Taint{} + for _, t := range node.Spec.Taints { + currentByKey[t.Key] = t } - slog.Info("node labels patched", "count", len(labels)) + for key, tmpl := range c.taintTemplates { + effectBuf := &bytes.Buffer{} + if err := tmpl.effect.Execute(effectBuf, c.discoveryData); err != nil { + slog.Warn("could not render effect template", "taint", key, "error", err) + continue + } - return nil + effect := corev1.TaintEffect(strings.TrimSpace(effectBuf.String())) + if effect == "" { + if _, existed := currentByKey[key]; existed { + deleteKeys = append(deleteKeys, key) + slog.Info("taint removed", "key", key) + } + continue + } + + if !isValidTaintEffect(effect) { + slog.Warn("invalid taint effect", "taint", key, "effect", effect) + continue + } + + valueBuf := &bytes.Buffer{} + if err := tmpl.value.Execute(valueBuf, c.discoveryData); err != nil { + slog.Warn("could not render value template", "taint", key, "error", err) + continue + } + + taint := corev1.Taint{Key: key, Value: sanitizeLabelValue(valueBuf.String()), Effect: effect} + if existing, ok := currentByKey[key]; ok && existing.Value == taint.Value && existing.Effect == taint.Effect { + continue + } + + upsert = append(upsert, taint) + slog.Info("taint changed", "key", key, "value", taint.Value, "effect", taint.Effect) + } + + return upsert, deleteKeys +} + +func isValidTaintEffect(effect corev1.TaintEffect) bool { + switch effect { + case corev1.TaintEffectNoSchedule, corev1.TaintEffectPreferNoSchedule, corev1.TaintEffectNoExecute: + return true + } + return false } diff --git a/internal/controller/controller_test.go b/internal/controller/controller_test.go new file mode 100644 index 0000000..3a77aa5 --- /dev/null +++ b/internal/controller/controller_test.go @@ -0,0 +1,250 @@ +package controller + +import ( + "reflect" + "sort" + "testing" + + "github.com/enix/topomatik/internal/config" + corev1 "k8s.io/api/core/v1" +) + +func newTestController(t *testing.T, templates map[string]config.TaintTemplate, data map[string]map[string]string) *Controller { + t.Helper() + c, err := New(nil, nil, nil, templates) + if err != nil { + t.Fatalf("New: %v", err) + } + c.discoveryData = data + return c +} + +func TestComputeTaintOps(t *testing.T) { + tests := []struct { + name string + templates map[string]config.TaintTemplate + data map[string]map[string]string + nodeTaints []corev1.Taint + wantUpsert []corev1.Taint + wantDelete []string + }{ + { + name: "no templates returns no ops", + nodeTaints: []corev1.Taint{ + {Key: "foo", Value: "bar", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + { + name: "adds new managed taint", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "{{ .hostname.zone }}", Effect: "NoSchedule"}, + }, + data: map[string]map[string]string{ + "hostname": {"zone": "eu-west"}, + }, + wantUpsert: []corev1.Taint{ + {Key: "specialized", Value: "eu-west", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + { + name: "no-op when managed taint already matches", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "{{ .hostname.zone }}", Effect: "NoSchedule"}, + }, + data: map[string]map[string]string{ + "hostname": {"zone": "eu-west"}, + }, + nodeTaints: []corev1.Taint{ + {Key: "specialized", Value: "eu-west", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + { + name: "updates managed taint when value changes", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "{{ .hostname.zone }}", Effect: "NoSchedule"}, + }, + data: map[string]map[string]string{ + "hostname": {"zone": "eu-west"}, + }, + nodeTaints: []corev1.Taint{ + {Key: "specialized", Value: "us-east", Effect: corev1.TaintEffectNoSchedule}, + }, + wantUpsert: []corev1.Taint{ + {Key: "specialized", Value: "eu-west", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + { + name: "updates managed taint when effect changes", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "fixed", Effect: "NoExecute"}, + }, + nodeTaints: []corev1.Taint{ + {Key: "specialized", Value: "fixed", Effect: corev1.TaintEffectNoSchedule}, + }, + wantUpsert: []corev1.Taint{ + {Key: "specialized", Value: "fixed", Effect: corev1.TaintEffectNoExecute}, + }, + }, + { + name: "templated effect renders from discovery data", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "fixed", Effect: "{{ .hostname.effect }}"}, + }, + data: map[string]map[string]string{ + "hostname": {"effect": "PreferNoSchedule"}, + }, + wantUpsert: []corev1.Taint{ + {Key: "specialized", Value: "fixed", Effect: corev1.TaintEffectPreferNoSchedule}, + }, + }, + { + name: "ignores unmanaged taints", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "fixed", Effect: "NoSchedule"}, + }, + nodeTaints: []corev1.Taint{ + {Key: "node.kubernetes.io/unreachable", Value: "", Effect: corev1.TaintEffectNoExecute}, + {Key: "specialized", Value: "old", Effect: corev1.TaintEffectNoSchedule}, + }, + wantUpsert: []corev1.Taint{ + {Key: "specialized", Value: "fixed", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + { + name: "sanitizes rendered values", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "@@@foo+bar.foobar----.", Effect: "NoSchedule"}, + }, + wantUpsert: []corev1.Taint{ + {Key: "specialized", Value: "foo_bar.foobar", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + { + name: "value render error skips taint", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "{{ .nope.value }}", Effect: "NoSchedule"}, + }, + data: map[string]map[string]string{}, + nodeTaints: []corev1.Taint{ + {Key: "specialized", Value: "old", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + { + name: "value render error with no existing taint is a no-op", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "{{ .nope.value }}", Effect: "NoSchedule"}, + }, + data: map[string]map[string]string{}, + }, + { + name: "effect render error skips taint", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "fixed", Effect: "{{ .nope.effect }}"}, + }, + data: map[string]map[string]string{}, + nodeTaints: []corev1.Taint{ + {Key: "specialized", Value: "old", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + { + name: "invalid rendered effect skips taint", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "fixed", Effect: "Bogus"}, + }, + nodeTaints: []corev1.Taint{ + {Key: "specialized", Value: "old", Effect: corev1.TaintEffectNoSchedule}, + }, + }, + { + name: "empty effect removes existing managed taint", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "fixed", Effect: ""}, + }, + nodeTaints: []corev1.Taint{ + {Key: "specialized", Value: "old", Effect: corev1.TaintEffectNoSchedule}, + }, + wantDelete: []string{"specialized"}, + }, + { + name: "empty effect from template removes existing managed taint", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "fixed", Effect: "{{ .hostname.effect }}"}, + }, + data: map[string]map[string]string{ + "hostname": {"effect": ""}, + }, + nodeTaints: []corev1.Taint{ + {Key: "specialized", Value: "old", Effect: corev1.TaintEffectNoSchedule}, + }, + wantDelete: []string{"specialized"}, + }, + { + name: "empty effect with no existing taint is a no-op", + templates: map[string]config.TaintTemplate{ + "specialized": {Value: "fixed", Effect: ""}, + }, + }, + { + name: "multiple managed upserts", + templates: map[string]config.TaintTemplate{ + "a": {Value: "x", Effect: "NoSchedule"}, + "b": {Value: "y", Effect: "NoExecute"}, + }, + wantUpsert: []corev1.Taint{ + {Key: "a", Value: "x", Effect: corev1.TaintEffectNoSchedule}, + {Key: "b", Value: "y", Effect: corev1.TaintEffectNoExecute}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := newTestController(t, tt.templates, tt.data) + node := &corev1.Node{Spec: corev1.NodeSpec{Taints: tt.nodeTaints}} + + gotUpsert, gotDelete := c.computeTaintOps(node) + + if !equalTaints(gotUpsert, tt.wantUpsert) { + t.Errorf("upsert mismatch\n got: %+v\nwant: %+v", gotUpsert, tt.wantUpsert) + } + if !equalStrings(gotDelete, tt.wantDelete) { + t.Errorf("delete keys mismatch\n got: %+v\nwant: %+v", gotDelete, tt.wantDelete) + } + }) + } +} + +func equalTaints(a, b []corev1.Taint) bool { + if len(a) != len(b) { + return false + } + if len(a) == 0 { + return true + } + return reflect.DeepEqual(sortTaints(a), sortTaints(b)) +} + +func sortTaints(taints []corev1.Taint) []corev1.Taint { + out := append([]corev1.Taint(nil), taints...) + sort.Slice(out, func(i, j int) bool { + if out[i].Key != out[j].Key { + return out[i].Key < out[j].Key + } + return out[i].Effect < out[j].Effect + }) + return out +} + +func equalStrings(a, b []string) bool { + if len(a) != len(b) { + return false + } + if len(a) == 0 { + return true + } + x := append([]string(nil), a...) + y := append([]string(nil), b...) + sort.Strings(x) + sort.Strings(y) + return reflect.DeepEqual(x, y) +}