Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 31 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ spec:
- [x] Local file / HTTP discovery engine #3
- [x] Hostname / node name discovery engine #4
- [x] Network config discovery engine #5
- [ ] Taint management #6
- [x] Taint management #6
- [ ] Prometheus Exporter #7
- [ ] Better RBAC / admission webhook #8

Expand Down Expand Up @@ -114,6 +114,36 @@ The [Sprig library](http://masterminds.github.io/sprig/) is available for advanc
>
> Example: "@@@foo+bar.foobar----." will be rendered as "foo_bar.foobar".

### Taint Templates

The `taintTemplates` section defines which Kubernetes taints Topomatik will manage. The map is keyed by the taint key; each entry has a `value` and an `effect`, both Go templates (same engine and Sprig functions as label templates).

```yaml
taintTemplates:
topology.plaffitt.kubernetes.io/specialized:
value: "{{ .hostname.zone }}"
effect: NoSchedule
```

Allowed rendered effects: `NoSchedule`, `PreferNoSchedule`, `NoExecute`. An effect that renders to an unsupported value preserves the existing taint on the node and logs a warning.

Rendered values are sanitized using the same rules as label values (see the note above).

If `effect` renders to an empty string, the taint is not applied; if it already exists on the node it will be removed. This lets you conditionally manage a taint, e.g.:

```yaml
taintTemplates:
topology.plaffitt.kubernetes.io/specialized:
value: "{{ .hostname.zone }}"
effect: '{{ if eq .hostname.zone "eu-west" }}NoSchedule{{ end }}'
```

> [!NOTE]
> Conditional management of labels is driven by an empty rendered `value`, but for taints it is driven by an empty rendered `effect`. A taint with an empty `value` is still a valid Kubernetes taint, whereas an empty `effect` is the natural signal that the taint should not exist.

> [!NOTE]
> Topomatik only manages taints whose key appears in `taintTemplates`. Taints set by other controllers or by the user (e.g. `kubectl taint`) on different keys are preserved on every reconciliation. Removing an entry from `taintTemplates` does not remove the corresponding taint from the node; remove it manually with `kubectl taint nodes <name> <key>-` or render the `effect` to an empty string.

### Auto-Discovery Engine Configuration

Each auto-discovery engine has its own configuration section.
Expand Down
2 changes: 1 addition & 1 deletion cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,7 @@ func main() {

scheduler := schedulers.NewSometimesWithDebounceChannel(config.MinimumReconciliationInterval)

ctrl, err := controller.New(k8sClientset, scheduler, config.LabelTemplates)
ctrl, err := controller.New(k8sClientset, scheduler, config.LabelTemplates, config.TaintTemplates)
if err != nil {
panic(err)
}
Expand Down
10 changes: 8 additions & 2 deletions internal/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,9 @@ import (
)

type Config struct {
LabelTemplates map[string]string `yaml:"labelTemplates"`
MinimumReconciliationInterval time.Duration `yaml:"minimumReconciliationInterval"`
LabelTemplates map[string]string `yaml:"labelTemplates"`
TaintTemplates map[string]TaintTemplate `yaml:"taintTemplates"`
MinimumReconciliationInterval time.Duration `yaml:"minimumReconciliationInterval"`

LLDP EngineConfig[lldp.Config] `yaml:"lldp"`
Files files.Config `yaml:"files" validate:"dive"`
Expand All @@ -27,6 +28,11 @@ type Config struct {
Network EngineConfig[network.Config] `yaml:"network"`
}

type TaintTemplate struct {
Value string `yaml:"value"`
Effect string `yaml:"effect"`
}

type EngineConfig[T any] struct {
Config T `yaml:",inline"`
Enabled bool `yaml:"enabled"`
Expand Down
152 changes: 130 additions & 22 deletions internal/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (

"github.com/Masterminds/sprig"
"github.com/enix/topomatik/internal/autodiscovery"
"github.com/enix/topomatik/internal/config"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
Expand All @@ -27,20 +28,27 @@ type ReconciliationScheduler interface {
C() <-chan struct{}
}

type taintTemplate struct {
value *template.Template
effect *template.Template
}

type Controller struct {
nodeName string
clientset *kubernetes.Clientset
labelTemplates map[string]*template.Template
taintTemplates map[string]*taintTemplate
engines []*autodiscovery.Engine
discoveryData map[string]map[string]string
scheduler ReconciliationScheduler
}

func New(clientset *kubernetes.Clientset, scheduler ReconciliationScheduler, labelTemplates map[string]string) (*Controller, error) {
func New(clientset *kubernetes.Clientset, scheduler ReconciliationScheduler, labelTemplates map[string]string, taintTemplates map[string]config.TaintTemplate) (*Controller, error) {
controller := &Controller{
nodeName: os.Getenv("NODE_NAME"),
clientset: clientset,
labelTemplates: map[string]*template.Template{},
taintTemplates: map[string]*taintTemplate{},
engines: []*autodiscovery.Engine{},
discoveryData: map[string]map[string]string{},
scheduler: scheduler,
Expand All @@ -53,6 +61,21 @@ func New(clientset *kubernetes.Clientset, scheduler ReconciliationScheduler, lab
}
}

for key, t := range taintTemplates {
value := template.New("taint:" + key + ":value").Funcs(sprig.FuncMap()).Option("missingkey=error")
if _, err := value.Parse(t.Value); err != nil {
return nil, err
}
effect := template.New("taint:" + key + ":effect").Funcs(sprig.FuncMap()).Option("missingkey=error")
if _, err := effect.Parse(t.Effect); err != nil {
return nil, err
}
controller.taintTemplates[key] = &taintTemplate{
value: value,
effect: effect,
}
}

return controller, nil
}

Expand Down Expand Up @@ -119,6 +142,13 @@ func (c *Controller) watchNode() error {
return nil
}

func sanitizeLabelValue(value string) string {
sanitized := labelRegexp.ReplaceAllString(value, "_")
return strings.TrimFunc(sanitized, func(r rune) bool {
return strings.Contains("_.-", string(r))
})
}

func (c *Controller) reconcileNode() error {
slog.Debug("reconciling node")
node, err := c.clientset.CoreV1().Nodes().Get(context.Background(), c.nodeName, metav1.GetOptions{})
Expand All @@ -127,18 +157,39 @@ func (c *Controller) reconcileNode() error {
return nil
}

labels := c.computeLabelPatch(node)
upsertTaints, deleteTaintKeys := c.computeTaintOps(node)

if len(labels) == 0 && len(upsertTaints) == 0 && len(deleteTaintKeys) == 0 {
slog.Debug("no changes detected")
return nil
}

patchBytes, err := buildNodeStrategicMergePatch(labels, upsertTaints, deleteTaintKeys)
if err != nil {
return fmt.Errorf("failed to marshal patch: %w", err)
}

_, err = c.clientset.CoreV1().Nodes().Patch(context.Background(), node.Name, types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{})
if err != nil {
return fmt.Errorf("could not patch node %s: %w", c.nodeName, err)
}

slog.Info("node patched", "labels", len(labels), "taintsUpserted", len(upsertTaints), "taintsDeleted", len(deleteTaintKeys))

return nil
}

func (c *Controller) computeLabelPatch(node *corev1.Node) map[string]any {
labels := map[string]any{}

for label, tmpl := range c.labelTemplates {
value := &bytes.Buffer{}
err := tmpl.Execute(value, c.discoveryData)
if err != nil {
if err := tmpl.Execute(value, c.discoveryData); err != nil {
slog.Warn("could not render template", "label", label, "error", err)
continue
}
sanitizedValue := labelRegexp.ReplaceAllString(value.String(), "_")
sanitizedValue = strings.TrimFunc(sanitizedValue, func(r rune) bool {
return strings.Contains("_.-", string(r))
})
sanitizedValue := sanitizeLabelValue(value.String())

currentValue, exists := node.Labels[label]
switch {
Expand All @@ -151,27 +202,84 @@ func (c *Controller) reconcileNode() error {
}
}

if len(labels) == 0 {
slog.Debug("no label changes detected")
return nil
return labels
}

func buildNodeStrategicMergePatch(labels map[string]any, upsertTaints []corev1.Taint, deleteTaintKeys []string) ([]byte, error) {
patch := map[string]any{}
if len(labels) > 0 {
patch["metadata"] = map[string]any{"labels": labels}
}

patch := map[string]any{
"metadata": map[string]any{
"labels": labels,
},
if len(upsertTaints) > 0 || len(deleteTaintKeys) > 0 {
taints := make([]any, 0, len(upsertTaints)+len(deleteTaintKeys))
for _, t := range upsertTaints {
taints = append(taints, t)
}

for _, key := range deleteTaintKeys {
taints = append(taints, map[string]any{"key": key, "$patch": "delete"})
}

patch["spec"] = map[string]any{"taints": taints}
}
patchBytes, err := json.Marshal(patch)
if err != nil {
return fmt.Errorf("failed to marshal patch: %w", err)

return json.Marshal(patch)
}

func (c *Controller) computeTaintOps(node *corev1.Node) (upsert []corev1.Taint, deleteKeys []string) {
if len(c.taintTemplates) == 0 {
return nil, nil
}

_, err = c.clientset.CoreV1().Nodes().Patch(context.Background(), node.Name, types.MergePatchType, patchBytes, metav1.PatchOptions{})
if err != nil {
return fmt.Errorf("could not patch node %s: %w", c.nodeName, err)
currentByKey := map[string]corev1.Taint{}
for _, t := range node.Spec.Taints {
currentByKey[t.Key] = t
}

slog.Info("node labels patched", "count", len(labels))
for key, tmpl := range c.taintTemplates {
effectBuf := &bytes.Buffer{}
if err := tmpl.effect.Execute(effectBuf, c.discoveryData); err != nil {
slog.Warn("could not render effect template", "taint", key, "error", err)
continue
}

return nil
effect := corev1.TaintEffect(strings.TrimSpace(effectBuf.String()))
if effect == "" {
if _, existed := currentByKey[key]; existed {
deleteKeys = append(deleteKeys, key)
slog.Info("taint removed", "key", key)
}
continue
}

if !isValidTaintEffect(effect) {
slog.Warn("invalid taint effect", "taint", key, "effect", effect)
continue
}

valueBuf := &bytes.Buffer{}
if err := tmpl.value.Execute(valueBuf, c.discoveryData); err != nil {
slog.Warn("could not render value template", "taint", key, "error", err)
continue
}

taint := corev1.Taint{Key: key, Value: sanitizeLabelValue(valueBuf.String()), Effect: effect}
if existing, ok := currentByKey[key]; ok && existing.Value == taint.Value && existing.Effect == taint.Effect {
continue
}

upsert = append(upsert, taint)
slog.Info("taint changed", "key", key, "value", taint.Value, "effect", taint.Effect)
}

return upsert, deleteKeys
}

func isValidTaintEffect(effect corev1.TaintEffect) bool {
switch effect {
case corev1.TaintEffectNoSchedule, corev1.TaintEffectPreferNoSchedule, corev1.TaintEffectNoExecute:
return true
}
return false
}
Loading
Loading