Add route-controller DaemonSet to prevent routing loops on local clusters

The route-controller runs as a DaemonSet on every node with host networking
and NET_ADMIN privileges. It discovers the cluster's service CIDRs (via the
SERVICE_CIDRS env var or the Kubernetes ServiceCIDR API on k8s 1.33+) and
installs iptables FORWARD chain DROP rules for each CIDR at startup. Any
forwarded packet destined for an IP with no active kube-proxy DNAT rule is
dropped rather than escaping via the node's default route, breaking the
routing loop that Telepresence can cause on local clusters.

A kernel RTN_BLACKHOLE route for the subnet was considered but rejected: it
fails connect()/sendmsg() before any iptables hook fires, which would break
kube-apiserver → mutating-webhook calls. The FORWARD chain only affects
forwarded (pod) traffic, not locally-generated host traffic, so active
services are unaffected because kube-proxy DNAT rewrites the destination
before the FORWARD chain is reached.

The DaemonSet is automatically enabled when image.registry is "local" or
starts with "localhost:", which are the conventional registry values for
local clusters (Kind, minikube, k3d, Docker Desktop). It can be
force-enabled or force-disabled with routeController.enabled=true/false
(null = auto-detect, the default).

Signed-off-by: Thomas Hallgren <thomas@tada.se>

Author: thallgren

CHANGELOG.yml

@@ -52,6 +52,15 @@ items:
           the need for elevated privileges when using Telepresence. Currently available for amd64 architecture only
           due to dependency constraints.
         docs: install/client
+      - type: feature
+        title: Add route-controller DaemonSet to prevent routing loops on local clusters
+        body: >-
+          A new optional <code>route-controller</code> DaemonSet can be deployed alongside the traffic-manager
+          on local Kubernetes clusters (Kind, minikube, k3d, Docker Desktop) to prevent routing loops caused by
+          deleted or non-existent service ClusterIPs. It installs an iptables <code>FORWARD</code> chain
+          <code>DROP</code> rule for the service CIDR on every node, and adds per-IP kernel blackhole routes
+          when a Service is deleted. Enable it with <code>routeController.enabled=true</code> in the Helm chart.
+        docs: reference/route-controller
       - type: feature
         title: Automatic cache cleanup on version change
         body: >-

build-aux/docker/images/Dockerfile.routecontroller

@@ -0,0 +1,40 @@
+# syntax = docker/dockerfile:1.3
+
+# Copyright 2024 Datawire. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM --platform=$BUILDPLATFORM golang:alpine AS routecontroller-build
+
+WORKDIR telepresence
+COPY go.mod go.sum .
+COPY cmd/routecontroller/ cmd/routecontroller/
+COPY pkg/ pkg/
+COPY build-output/version.txt .
+
+ARG TARGETOS
+ARG TARGETARCH
+
+RUN \
+    --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    GOOS=$TARGETOS GOARCH=$TARGETARCH go build -o /usr/local/bin/route-controller -trimpath -ldflags="-s -w" ./cmd/routecontroller/
+
+FROM alpine AS routecontroller
+
+RUN apk add --no-cache ca-certificates iptables ip6tables
+
+COPY --from=routecontroller-build /usr/local/bin/route-controller /usr/local/bin/route-controller
+
+ENTRYPOINT ["/usr/local/bin/route-controller"]
+CMD []

build-aux/main.mk

@@ -326,8 +326,20 @@ save-tel2-image: tel2-image
 push-client-image: client-image ## (Build) Push the client container image to $(TELEPRESENCE_REGISTRY)
 	docker push $(CLIENT_IMAGE_FQN)
 
+ROUTECONTROLLER_IMAGE_FQN=$(TELEPRESENCE_REGISTRY)/route-controller:$(TELEPRESENCE_SEMVER)
+
+.PHONY: routecontroller-image
+routecontroller-image: images-deps  ## (Build) Build the route-controller DaemonSet image
+	$(eval PLATFORM_ARG := $(if $(TELEPRESENCE_ROUTECONTROLLER_IMAGE_PLATFORM), --platform=$(TELEPRESENCE_ROUTECONTROLLER_IMAGE_PLATFORM),))
+	docker build $(PLATFORM_ARG) --target routecontroller --tag route-controller --tag $(ROUTECONTROLLER_IMAGE_FQN) \
+	    -f build-aux/docker/images/Dockerfile.routecontroller .
+
+.PHONY: push-routecontroller-image
+push-routecontroller-image: routecontroller-image ## (Build) Push the route-controller DaemonSet image to $(TELEPRESENCE_REGISTRY)
+	docker push $(ROUTECONTROLLER_IMAGE_FQN)
+
 .PHONY: push-images
-push-images: push-tel2-image push-client-image
+push-images: push-tel2-image push-client-image push-routecontroller-image
 
 .PHONY: helm-chart
 helm-chart: $(BUILDDIR)/telepresence-oss-chart.tgz

charts/telepresence-oss/templates/routecontroller-daemonset.yaml

@@ -0,0 +1,43 @@
+{{- $localCluster := or (eq .Values.image.registry "local") (hasPrefix "localhost:" .Values.image.registry) }}
+{{- $rcEnabled := ternary $localCluster .Values.routeController.enabled (kindIs "invalid" .Values.routeController.enabled) }}
+{{- if and .Values.routeController $rcEnabled }}
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: route-controller
+  namespace: {{ include "traffic-manager.namespace" $ }}
+  labels:
+    {{ include "telepresence.labels" $ | nindent 4 }}
+spec:
+  selector:
+    matchLabels:
+      app: route-controller
+  template:
+    metadata:
+      labels:
+        app: route-controller
+    spec:
+      hostNetwork: true
+      dnsPolicy: ClusterFirstWithHostNet
+      serviceAccountName: route-controller
+      containers:
+        - name: route-controller
+          image: "{{ coalesce .Values.routeController.image.registry .Values.image.registry }}/{{ .Values.routeController.image.name }}:{{ .Chart.AppVersion }}"
+          imagePullPolicy: {{ coalesce .Values.routeController.image.pullPolicy .Values.image.pullPolicy }}
+          env:
+            - name: LOG_LEVEL
+              value: {{ coalesce .Values.routeController.logLevel .Values.logLevel }}
+            {{- with .Values.routeController.serviceCIDRs }}
+            - name: SERVICE_CIDRS
+              value: {{ join "," . }}
+            {{- end }}
+          securityContext:
+            capabilities:
+              add: ["NET_ADMIN"]
+          resources:
+            requests:
+              cpu: 10m
+              memory: 32Mi
+            limits:
+              memory: 64Mi
+{{- end }}

charts/telepresence-oss/templates/routecontroller-rbac.yaml

@@ -0,0 +1,32 @@
+{{- $localCluster := or (eq .Values.image.registry "local") (hasPrefix "localhost:" .Values.image.registry) }}
+{{- $rcEnabled := ternary $localCluster .Values.routeController.enabled (kindIs "invalid" .Values.routeController.enabled) }}
+{{- if and .Values.routeController $rcEnabled }}
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: route-controller
+  namespace: {{ include "traffic-manager.namespace" $ }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: route-controller
+rules:
+  - apiGroups: ["networking.k8s.io"]
+    resources: ["servicecidrs"]
+    verbs: ["get", "list"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: route-controller
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: route-controller
+subjects:
+  - kind: ServiceAccount
+    name: route-controller
+    namespace: {{ include "traffic-manager.namespace" $ }}
+{{- end }}

charts/telepresence-oss/values.schema.yaml

@@ -507,6 +507,53 @@ properties:
     description: Number or replicas for the traffic-manager. The Traffic Manager only support running with one replica at the moment.
     const: 1
 
+  routeController:
+    description: >-
+      Configuration for the route-controller DaemonSet that installs blackhole routes for deleted service ClusterIPs
+      to prevent routing loops on local clusters.
+    type: object
+    additionalProperties: false
+    properties:
+      enabled:
+        description: >-
+          Enable or disable the route-controller DaemonSet. When null (the default), the
+          route-controller is automatically enabled for local clusters, detected by
+          image.registry being "local" or starting with "localhost:". Set to true to
+          force-enable or false to force-disable regardless of registry.
+        anyOf:
+          - type: "null"
+          - type: boolean
+      image:
+        type: object
+        additionalProperties: false
+        properties:
+          name:
+            description: The name of the route-controller image
+            type: string
+          pullPolicy:
+            description: Pull policy for the route-controller image. Empty string inherits from image.pullPolicy.
+            anyOf:
+              - type: string
+                const: ""
+              - $ref: "#/$defs/pullPolicy"
+          registry:
+            description: The registry for the route-controller image
+            type: string
+      logLevel:
+        description: Log level for the route-controller. Empty string inherits from logLevel.
+        anyOf:
+          - type: string
+            const: ""
+          - $ref: "#/$defs/logLevel"
+      serviceCIDRs:
+        description: >-
+          Service CIDRs for which subnet-level blackhole routes are installed at startup.
+          If empty, the controller queries the ServiceCIDR API (k8s >= 1.33) automatically.
+          For older clusters, set this explicitly (e.g. ["10.96.0.0/12"]).
+        type: array
+        items:
+          type: string
+
   resources:
     description: Define resource requests and limits for the Traffic Manger
     $ref: "#/$defs/resourceRequirements"

charts/telepresence-oss/values.yaml

@@ -198,3 +198,31 @@ intercept:
   # Once this timeout is exceeded, the intercept no longer blocks conflicting intercepts and may be
   # automatically removed when another client attempts to create a conflicting intercept.
   inactiveBlockTimeout: 10m
+
+################################################################################
+## Route Controller Configuration
+################################################################################
+# routeController installs a DaemonSet that watches for Service deletions and
+# adds temporary blackhole routes on each node for deleted ClusterIPs, preventing
+# routing loops on local clusters (Kind, minikube, k3d, Docker Desktop) where
+# deleted service IPs fall through to the node's default route.
+routeController:
+  # enabled controls the route-controller DaemonSet:
+  #   null (default) - auto-enable when image.registry is "local" or starts with "localhost:"
+  #   true           - always enable
+  #   false          - always disable, even on local clusters
+  enabled: ~
+  image:
+    # registry and pullPolicy default to the traffic-manager image settings when not set
+    registry: ""
+    name: route-controller
+    pullPolicy: ""
+  # logLevel defaults to the traffic-manager logLevel when not set
+  logLevel: ""
+  # serviceCIDRs is a list of service CIDRs (e.g. ["10.96.0.0/12"]) for which the
+  # route-controller installs subnet-level blackhole routes at startup. These are
+  # overridden by kube-proxy for active services and catch any non-existent IP in
+  # the range without waiting for a deletion event.
+  # If empty, the controller queries the ServiceCIDR API (k8s >= 1.33) and falls
+  # back to per-IP blackhole routes on deletion for older clusters.
+  serviceCIDRs: []

cmd/routecontroller/main.go

@@ -0,0 +1,144 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"os"
+	"strings"
+
+	"github.com/coreos/go-iptables/iptables"
+	meta "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+
+	"github.com/telepresenceio/clog"
+	tplog "github.com/telepresenceio/telepresence/v2/pkg/log"
+	"github.com/telepresenceio/telepresence/v2/pkg/sigctx"
+)
+
+// subnetRule tracks an iptables FORWARD DROP rule installed for a service CIDR.
+type subnetRule struct {
+	ipt  *iptables.IPTables
+	cidr string
+}
+
+func main() {
+	ctx := context.Background()
+	logLevel := os.Getenv("LOG_LEVEL")
+	ctx = tplog.MakeBaseLogger(ctx, os.Stdout, logLevel)
+
+	if err := sigctx.DoWithSignalHandler(ctx, run); err != nil {
+		clog.Error(ctx, err)
+		os.Exit(1)
+	}
+}
+
+func run(ctx context.Context) error {
+	cfg, err := rest.InClusterConfig()
+	if err != nil {
+		return fmt.Errorf("failed to get in-cluster config: %w", err)
+	}
+
+	cs, err := kubernetes.NewForConfig(cfg)
+	if err != nil {
+		return fmt.Errorf("failed to create clientset: %w", err)
+	}
+
+	rules := installSubnetBlackholes(ctx, cs)
+	defer removeSubnetBlackholes(ctx, rules)
+
+	clog.Info(ctx, "Route controller started")
+	<-ctx.Done()
+	return nil
+}
+
+// discoverServiceCIDRs returns the service CIDRs for which iptables FORWARD DROP rules
+// will be installed. It first checks the SERVICE_CIDRS environment variable
+// (comma-separated list of CIDRs). If that is not set, it queries the Kubernetes
+// ServiceCIDR API (available in k8s 1.33+). If neither is available it returns nil and
+// no subnet-level protection is installed; set SERVICE_CIDRS explicitly in that case.
+func discoverServiceCIDRs(ctx context.Context, cs *kubernetes.Clientset) []string {
+	if envCIDRs := os.Getenv("SERVICE_CIDRS"); envCIDRs != "" {
+		cidrs := strings.Split(envCIDRs, ",")
+		clog.Infof(ctx, "Using service CIDRs from SERVICE_CIDRS env: %v", cidrs)
+		return cidrs
+	}
+
+	list, err := cs.NetworkingV1().ServiceCIDRs().List(ctx, meta.ListOptions{})
+	if err != nil {
+		clog.Warnf(ctx, "ServiceCIDR API unavailable: %v; set SERVICE_CIDRS to enable subnet blackholing", err)
+		return nil
+	}
+	var cidrs []string
+	for _, item := range list.Items {
+		cidrs = append(cidrs, item.Spec.CIDRs...)
+	}
+	clog.Infof(ctx, "Discovered service CIDRs from cluster API: %v", cidrs)
+	return cidrs
+}
+
+// installSubnetBlackholes installs iptables FORWARD chain DROP rules for each service CIDR.
+//
+// An iptables rule in the FORWARD chain (rather than a kernel blackhole route) is used
+// because RTN_BLACKHOLE routes fail connect()/sendmsg() at the socket level before
+// any iptables hook can fire, which breaks locally-generated traffic such as
+// kube-apiserver → mutating-webhook calls.
+//
+// The FORWARD chain only affects traffic forwarded through the host (i.e. pod traffic via
+// veth pairs). Locally-generated host traffic is never subject to the FORWARD chain.
+//
+// For active services, kube-proxy's PREROUTING DNAT fires before the FORWARD chain and
+// rewrites the destination from ClusterIP to a pod IP, so the DROP rule (which matches
+// on the original service CIDR) does not apply. For deleted or never-assigned ClusterIPs
+// no DNAT rule exists, the FORWARD chain sees the original ClusterIP, and the DROP fires.
+func installSubnetBlackholes(ctx context.Context, cs *kubernetes.Clientset) []subnetRule {
+	var rules []subnetRule
+	for _, cidr := range discoverServiceCIDRs(ctx, cs) {
+		cidr = strings.TrimSpace(cidr)
+		_, network, err := net.ParseCIDR(cidr)
+		if err != nil {
+			clog.Errorf(ctx, "Failed to parse service CIDR %q: %v", cidr, err)
+			continue
+		}
+
+		proto := iptables.ProtocolIPv4
+		if network.IP.To4() == nil {
+			proto = iptables.ProtocolIPv6
+		}
+
+		ipt, err := iptables.NewWithProtocol(proto)
+		if err != nil {
+			clog.Errorf(ctx, "Failed to initialise iptables for %s: %v", network, err)
+			continue
+		}
+
+		exists, err := ipt.Exists("filter", "FORWARD", "-d", network.String(), "-j", "DROP")
+		if err != nil {
+			clog.Errorf(ctx, "Failed to check iptables FORWARD rule for %s: %v", network, err)
+			continue
+		}
+		if !exists {
+			if err := ipt.Insert("filter", "FORWARD", 1, "-d", network.String(), "-j", "DROP"); err != nil {
+				clog.Errorf(ctx, "Failed to add iptables FORWARD DROP for service CIDR %s: %v", network, err)
+				continue
+			}
+			clog.Infof(ctx, "Added iptables FORWARD DROP for service CIDR %s", network)
+		} else {
+			clog.Debugf(ctx, "iptables FORWARD DROP for %s already present", network)
+		}
+
+		rules = append(rules, subnetRule{ipt: ipt, cidr: network.String()})
+	}
+	return rules
+}
+
+func removeSubnetBlackholes(ctx context.Context, rules []subnetRule) {
+	for _, rule := range rules {
+		if err := rule.ipt.Delete("filter", "FORWARD", "-d", rule.cidr, "-j", "DROP"); err != nil {
+			clog.Debugf(ctx, "Failed to remove iptables FORWARD DROP for %s: %v", rule.cidr, err)
+		} else {
+			clog.Infof(ctx, "Removed iptables FORWARD DROP for service CIDR %s", rule.cidr)
+		}
+	}
+}