From 0ce60d1919b4869f5703032219d1c11522022645 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alby=20Hern=C3=A1ndez?= Date: Wed, 24 Jun 2026 18:44:52 +0100 Subject: [PATCH] feat: host-level data path performance tuning Add spec.host on EdgeNode to tune the VPS as a machine, independent of the proxy. Features: - spec.host.kernelMaxSocketBufferBytes (default 25MB): single knob wired to net.core.rmem_max/wmem_max and to SO_RCVBUF/SO_SNDBUF on the Envoy UDP listeners (UDP does not autotune; TCP is left to autotune). - spec.host.disableNicOffloads (default false): turn GRO/GSO off on the underlay NIC (auto-detected via netlink route lookup), applied natively over ethtool-netlink as a netdev section of the relay document. Changes: - render: drop prefer_gro, set enable_reuse_port on every listener. - agentconfig: optional netdev section in the desired-state document. - internal/netdev: native ethtool applier with an IO seam and a root-gated test. - rename the boot oneshot wg-relay.service to tunnel-boot.service (the WireGuard interface wg-relay is unchanged). --- .agents/ARCHITECTURE.md | 12 +- .agents/DESIGN_AND_RULES.md | 12 ++ api/v1alpha1/edgenode_types.go | 32 ++++ api/v1alpha1/zz_generated.deepcopy.go | 16 ++ cmd/tunnelctl/main.go | 5 +- .../tunnel.achetronic.com_edgenodes.yaml | 27 ++++ config/samples/tunnel_v1alpha1_edgenode.yaml | 19 +++ .../crds/tunnel.achetronic.com_edgenodes.yaml | 27 ++++ go.mod | 3 +- go.sum | 2 + internal/agentconfig/agentconfig.go | 12 ++ internal/agentconfig/agentconfig_test.go | 71 +++++++++ internal/agentconfig/agentconfig_types.go | 17 +++ internal/agentrun/agentrun.go | 28 +++- internal/agentrun/agentrun_test.go | 44 ++++++ internal/netdev/netdev.go | 139 ++++++++++++++++++ internal/netdev/netdev_kernel_test.go | 97 ++++++++++++ internal/netdev/netdev_test.go | 135 +++++++++++++++++ internal/planner/plan.go | 55 +++++-- internal/planner/plan_test.go | 103 +++++++++++++ internal/planner/plan_types.go | 8 + internal/provision/enroll.go | 80 +++++++--- internal/provision/enroll_test.go | 72 +++++---- internal/provision/teardown.go | 13 +- internal/render/envoy.go | 22 ++- internal/render/envoy_test.go | 79 ++++++++++ internal/render/envoy_validate_test.go | 1 + internal/render/render_types.go | 8 + internal/render/testdata/envoy_lds.golden | 11 +- .../testdata/envoy_lds_tls_mutual.golden | 4 + .../testdata/envoy_lds_tls_offload.golden | 4 + .../testdata/envoy_lds_tls_passthrough.golden | 4 + 32 files changed, 1075 insertions(+), 87 deletions(-) create mode 100644 internal/netdev/netdev.go create mode 100644 internal/netdev/netdev_kernel_test.go create mode 100644 internal/netdev/netdev_test.go diff --git a/.agents/ARCHITECTURE.md b/.agents/ARCHITECTURE.md index 3ed80fb..23a046c 100644 --- a/.agents/ARCHITECTURE.md +++ b/.agents/ARCHITECTURE.md @@ -71,13 +71,12 @@ Validation is split: CEL on the CRD enforces the spec shape (`tls` only on TCP b ### 4.1 EdgeNodeReconciler (sole writer to the machine and uplink) 1. **IPAM and keys**: Assigns tunnel IP by ordinal. Generates one WireGuard keypair per replica (stored in the `-uplink-keys` Secret). The VPS keypair is generated locally in-cluster with `wgctrl`/`wgtypes` and cached in the `-vps-key` Secret, so no command runs on the VPS to create it. -2. **SSH Enrollment**: Idempotent. Pushes the matching static `tunnelctl` binary to the VPS (read from the local `--tunnelctl-dir`: baked into the operator image, or a local build dir under `make run`; the right arch chosen from `uname -m`), plus the static Envoy binary; writes the relay `tunnelctl` desired-state document and applies it with `tunnelctl apply` (WireGuard via netlink, no `wireguard-tools`, no `wg-quick`, no download on the VPS). The only systemd units installed are the Envoy service and a `wg-relay` boot oneshot (`Type=oneshot`, `RemainAfterExit=yes`) that reapplies the relay document on boot so WireGuard survives a VPS reboot; the Envoy unit `Requires`/`After` it. Neither is a resident WireGuard daemon. It also writes the immutable Envoy bootstrap plus LDS/CDS files, any TLS material for edge-terminating bindings, and `state.json` with artifact hashes (relay document, tunnelctl binary hash, LDS, CDS, TLS). +2. **SSH Enrollment**: Idempotent. Pushes the matching static `tunnelctl` binary to the VPS (read from the local `--tunnelctl-dir`: baked into the operator image, or a local build dir under `make run`; the right arch chosen from `uname -m`), plus the static Envoy binary; writes the relay `tunnelctl` desired-state document and applies it with `tunnelctl apply` (WireGuard via netlink, no `wireguard-tools`, no `wg-quick`, no download on the VPS). The only systemd units installed are the Envoy service and the `tunnel-boot.service` boot oneshot (`Type=oneshot`, `RemainAfterExit=yes`) that reapplies the relay desired-state document on boot (WireGuard plus any `netdev` tuning) so the relay survives a VPS reboot; the Envoy unit `Requires`/`After` it. Neither is a resident daemon. The unit is named `tunnel-boot` because it reapplies the whole node document, not just WireGuard; the WireGuard *interface* it brings up is named `wg-relay`. It also writes the immutable Envoy bootstrap plus LDS/CDS files, any TLS material for edge-terminating bindings, and `state.json` with artifact hashes (relay document, tunnelctl binary hash, LDS, CDS, TLS). 3. **Uplink**: Creates/updates a StatefulSet with the uplink image, a ConfigMap holding the shared uplink `tunnelctl` desired-state document (WireGuard + nftables; each replica injects its own identity at runtime), and the headless Service the StatefulSet's `spec.serviceName` references (ClusterIP None, selector-only). 4. **Aggregate Render**: Lists all PortBindings that reference this EdgeNode, resolves Service targets to ClusterIPs, validates cross-binding port conflicts (and conflicts with the tunnel ListenPort), and renders the relay and uplink `tunnelctl` desired-state documents plus the Envoy LDS/CDS via `internal/planner.BuildPlan`. For bindings with `tls` in `offload`/`mutual`, the plan also declares the TLS material to install (the per-binding SDS document path on the VPS); the reconciler reads the referenced `kubernetes.io/tls` Secret, validates the required keys, renders a per-binding SDS document (server cert/key inline, plus `ca.crt` for `mutual`) and pushes it to the edge. 5. **Drift**: Re-render is event-driven plus a periodic safety net. The happy path requeues after `RequeueInterval` (default 3 min) for drift detection and status refresh, and also reacts to watches (the EdgeNode itself, the PortBindings that reference it, and the TLS Secrets referenced by its bindings). It compares `status.appliedConfigHash` (the plan hash folded with the TLS material hash, so cert rotations are visible) and the per-artifact hashes against the desired plan and the remote `state.json`; only re-pushes/restarts services when hashes differ, and reports health back into status. Envoy health is read via `systemctl is-active`; relay WireGuard health and the peer handshakes are read via `tunnelctl status` (JSON + exit code). Optimistic-concurrency conflicts on writes/status updates are turned into clean requeues, never swallowed. 6. **Binary delivery**: which Envoy release is installed is an operator-wide setting, the manager flag `--envoy-version` (default `controller.DefaultEnvoyVersion`); `provision.installEnvoyBinary` builds the download URL from version + architecture and reinstalls when Envoy is absent or a different version. `tunnelctl` is NOT downloaded on the VPS: the operator reads the static binary from `--tunnelctl-dir` (default `controller.DefaultTunnelctlDir`, baked into the operator image; overridden to a local build dir under `make run`) and pushes the arch-matching one over SSH, re-pushing only when its SHA-256 changes (tracked in `state.json`). The controller injects the Envoy version and the tunnelctl dir into the plan after `BuildPlan` (the planner stays pure). The operator stamps its own version into the binary via `internal/version.Version` (`-ldflags -X`), logged at startup. A running Envoy does not pick up a new binary until it is restarted (see the restart-envoy annotation). -7. **Teardown** (on deletion, guarded by a finalizer): runs `provision.Teardown` over SSH (deletes the `wg-relay` interface with `ip link del`, removes the `tunnelctl` binary, stops/disables both the Envoy service and the `wg-relay` boot oneshot and removes both systemd units, removes the Envoy config, the sysctl drop-in and `/etc/tunnel`) and then explicitly deletes the in-cluster uplink StatefulSet, ConfigMap, keys Secret and the cached `-vps-key` Secret. They are deleted explicitly (not via owner references) because they live in `spec.uplink.namespace`, which can differ from the EdgeNode namespace, and cross-namespace owner-reference GC is not allowed. Set the `tunnel.achetronic.com/skip-deprovision: "true"` annotation to skip the SSH teardown. - +7. **Teardown** (on deletion, guarded by a finalizer): runs `provision.Teardown` over SSH (deletes the `wg-relay` interface with `ip link del`, removes the `tunnelctl` binary, stops/disables both the Envoy service and the `tunnel-boot` boot oneshot and removes both systemd units, removes the Envoy config, the sysctl drop-in and `/etc/tunnel`) and then explicitly deletes the in-cluster uplink StatefulSet, ConfigMap, keys Secret and the cached `-vps-key` Secret. They are deleted explicitly (not via owner references) because they live in `spec.uplink.namespace`, which can differ from the EdgeNode namespace, and cross-namespace owner-reference GC is not allowed. Set the `tunnel.achetronic.com/skip-deprovision: "true"` annotation to skip the SSH teardown. #### 4.1.0 Operational annotations - `tunnel.achetronic.com/skip-deprovision: "true"` skips the SSH teardown on deletion (the in-cluster uplink resources are still removed). - `tunnel.achetronic.com/restart-envoy: "true"` makes the next reconcile restart Envoy on the VPS (via `provision.RestartEnvoy`, a systemctl restart that waits for the unit to become active) after enrollment, the way to apply a new `--envoy-version`. It is a one-shot: the reconciler consumes and removes the annotation in the same metadata update as the finalizer, before any status mutation, so it never clobbers the status write and never loops. A failed restart surfaces as `Ready=False`/`EnvoyRestartFailed`. @@ -85,6 +84,13 @@ Validation is split: CEL on the CRD enforces the spec shape (`tls` only on TCP b #### 4.1.1 TLS certificate rotation The EdgeNodeReconciler `Watches` Secrets. When a Secret referenced by some binding's `tls.secretRef` changes (e.g. cert-manager renews it), a map function (`mapSecretToEdgeNodes`) resolves which EdgeNodes consume it and enqueues them, so the renewed certificate is re-pushed to the VPS. Delivery is via file-based **SDS**: the operator writes `/etc/envoy/tls/.sds.yaml` (cert/key inline, plus the CA for `mutual`) with a single atomic `mv` into the `watched_directory` (`/etc/envoy/tls`) that Envoy monitors; Envoy hot-reloads just that secret gracefully, with no listener rebuild, no dropped connections and no restart. Only Secrets actually referenced by a TLS binding produce a requeue. The `PrivateKeyOnEdge` warning Event is emitted only when key material is genuinely (re)written this reconcile, not on every loop. +#### 4.1.2 Host-level performance tuning +The optional `spec.host` block tunes the VPS as a machine (kernel and NIC), independent of the proxy. The planner defaults and threads it; nothing here is per-listener and there is no generic escape-hatch. +- **`kernelMaxSocketBufferBytes` (socket-buffer ceiling, default 25MB).** One knob wired to two layers, so the user sets a single number. (1) The operator writes the kernel ceiling `net.core.rmem_max`/`wmem_max` into the sysctl drop-in `/etc/sysctl.d/99-tunnel.conf` over SSH during enrollment (alongside `ip_forward`), persisted natively across reboots like any sysctl drop-in. (2) The Envoy render sets `SO_RCVBUF`/`SO_SNDBUF` (SOL_SOCKET, `STATE_PREBIND`) on the UDP listeners only, up to that ceiling. TCP autotunes below the ceiling on its own, so it gets no explicit socket options; UDP does not autotune, so the explicit reclaim is what makes the ceiling effective for it. +- **`disableNicOffloads` (GRO/GSO toggle, default false).** Turns GRO and GSO off on the underlay NIC carrying the WireGuard transport, auto-detected via a netlink route lookup to a public IP (never the `wg-relay` tunnel device, never a hardcoded name). It is applied natively over ethtool-netlink by `tunnelctl` as the `netdev` section of the relay desired-state document (no `ethtool` CLI on the host), so the `tunnel-boot` oneshot reapplies it on reboot. Default off because disabling offloads degrades throughput for ordinary TCP/QUIC traffic by raising per-packet CPU; it is for encapsulated, non-uniform UDP (tunnel-in-tunnel) where GRO coalescing corrupts datagram boundaries and breaks Envoy's L4 datagram semantics. +- **Envoy render (no knob).** GRO is delegated to the OS (controlled at the NIC via `disableNicOffloads`), so the UDP listeners carry no `prefer_gro`. Every listener sets `enable_reuse_port: true` explicitly (honored for both TCP and UDP on Linux; the `reuse_port` field is deprecated). +- **Teardown does not restore live values.** Like `ip_forward`, teardown removes the persistence (the sysctl drop-in file and the `tunnel-boot` unit that reapplies the `netdev` section) and lets the live kernel/NIC values revert on the next reboot, rather than capturing and restoring prior state. + ### 4.2 PortBindingReconciler (independent of the EdgeNodeReconciler) PortBindings have their own reconciler with its own lifecycle; the two controllers share no code or memory and communicate exclusively through the API server (`status.appliedBindings` + an EdgeNode watch upstream, see the NOTE below). This reconciler does not speak SSH, does not touch the uplink, does not resolve targets or validate conflicts, and never writes to the EdgeNode; its sole job is publishing the PortBinding's conditions. Plan rebuilding is driven by the EdgeNodeReconciler itself: it `Watches` PortBindings with a map function (`mapPortBindingToEdgeNode`) that enqueues the referenced EdgeNode on every create, spec change and deletion (a generation predicate filters PortBinding status writes, which carry no plan input), so the aggregate plan always reflects the live set of bindings, including dropped listeners after a deletion. PortBindings carry no finalizer; if an object still has `tunnel.achetronic.com/portbinding-finalizer`, the reconciler removes it on deletion so the object is never stuck. Target resolution and cross-binding conflict validation (including binding-name uniqueness across all PortBindings of a node, since the name keys the Envoy listener and the SDS document path) happen inside `BuildPlan` during EdgeNode reconciliation. diff --git a/.agents/DESIGN_AND_RULES.md b/.agents/DESIGN_AND_RULES.md index 334b84b..d4e7de2 100644 --- a/.agents/DESIGN_AND_RULES.md +++ b/.agents/DESIGN_AND_RULES.md @@ -41,3 +41,15 @@ A TCP PortBinding may terminate or route TLS at the VPS via an optional `tls: { ## 6. Agents This `.agents/` folder contains the knowledge material the LLM must read to absorb context before altering the data path. + +## 7. Host-Level Tuning (Closed Decisions) + +- **Single knob for socket buffers:** `spec.host.kernelMaxSocketBufferBytes` provides a unified control for kernel `net.core.rmem_max`/`wmem_max` sysctls and Envoy UDP listener `SO_RCVBUF`/`SO_SNDBUF` socket options. This simplifies user configuration: one value effectively raises the socket buffer ceiling across both kernel and userspace for UDP, which does not autotune. TCP socket buffers autotune up to this ceiling, so no explicit Envoy TCP listener socket options are set. +- **Dedicated NIC offload toggle:** `spec.host.disableNicOffloads` offers a precise switch for GRO/GSO on the underlay physical NIC carrying the WireGuard transport. + - **Auto-detection:** The underlay NIC is auto-detected via `netlink route get` to the VPS's public IP. + - **Natively applied:** Offloads are managed using `ethtool-netlink` by `tunnelctl` as a `netdev` section within the relay desired-state document. This method ensures changes persist across reboots via the boot oneshot. + - **Default off:** The default `false` (offloads enabled) is chosen because disabling GRO/GSO degrades throughput for general TCP/QUIC traffic. It is intended for specific encapsulated UDP or tunnel-in-tunnel workloads where GRO can corrupt datagram boundaries and break L4 datagram semantics. + - **No per-listener granularity:** The offload setting applies to the entire underlay NIC, not per Envoy listener, reflecting its kernel-level nature. + - **No generic escape-hatch:** This specific toggle for GRO/GSO addresses a known issue in certain high-performance tunnel scenarios; a generic `ethtool` escape-hatch is not provided to prevent arbitrary kernel network changes. +- **Envoy `prefer_gro` absent, `enable_reuse_port:true` explicit:** GRO is delegated to the OS (and disabled at the NIC via `disableNicOffloads` when needed), so the operator-managed Envoy config carries no `prefer_gro`. Every listener sets `enable_reuse_port:true` (honored for both TCP and UDP on Linux; the `reuse_port` field is deprecated). +- **Teardown does not restore live values:** Consistent with `ip_forward` and other host-level settings, teardown removes the persistence mechanisms (sysctl drop-in, `netdev` section from the boot oneshot) for `kernelMaxSocketBufferBytes` and `disableNicOffloads`. Live kernel values are not restored; the host reverts to its defaults on the next reboot. diff --git a/api/v1alpha1/edgenode_types.go b/api/v1alpha1/edgenode_types.go index a01788a..aa5314e 100644 --- a/api/v1alpha1/edgenode_types.go +++ b/api/v1alpha1/edgenode_types.go @@ -163,6 +163,32 @@ type HealthCheckSpec struct { UnhealthyThreshold int32 `json:"unhealthyThreshold,omitempty"` } +// HostSpec tunes the VPS as a machine (kernel and NIC), independent of the +// proxy (edge). These knobs shape the data path at the operating-system level: +// the kernel socket-buffer ceiling and the underlay NIC offloads. They are +// applied to the host during enrollment and reconciliation and are orthogonal +// to spec.edge, which configures the Envoy proxy itself. +type HostSpec struct { + // KernelMaxSocketBufferBytes is the socket-buffer ceiling. It is threaded to + // both the kernel sysctls (net.core.rmem_max/wmem_max) and the + // SO_RCVBUF/SO_SNDBUF of Envoy's UDP listeners. UDP does not autotune, so + // this single knob raises the effective buffer for high-throughput UDP while + // TCP autotunes up to the same ceiling. Defaults to 26214400 (25MB). + // +kubebuilder:default=26214400 + // +kubebuilder:validation:Minimum=0 + // +optional + KernelMaxSocketBufferBytes int64 `json:"kernelMaxSocketBufferBytes,omitempty"` + + // DisableNicOffloads turns GRO/GSO off on the underlay NIC carrying the + // WireGuard transport. It defaults false (offloads enabled) because disabling + // them degrades throughput for ordinary TCP/QUIC traffic; enable it only for + // encapsulated/non-uniform UDP payloads where GRO coalescing can corrupt + // datagram boundaries. + // +kubebuilder:default=false + // +optional + DisableNicOffloads bool `json:"disableNicOffloads,omitempty"` +} + // EdgeNodeSpec defines the desired state of a EdgeNode: a single VPS acting // as a public relay plus the in-cluster uplink that terminates its tunnels. type EdgeNodeSpec struct { @@ -174,6 +200,12 @@ type EdgeNodeSpec struct { // +required SSH SSHSpec `json:"ssh"` + // Host tunes the VPS as a machine (kernel and NIC), independent of the proxy. + // Optional; sane defaults apply. Placed between SSH and Edge by convention: + // host-level OS tuning sits below the proxy configuration. + // +optional + Host HostSpec `json:"host,omitempty"` + // Edge configures the VPS-side edge proxy: how it health checks the uplink // replicas and fails over between them. Optional; sane defaults apply. // +optional diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index e165869..b1a6490 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -96,6 +96,7 @@ func (in *EdgeNodeList) DeepCopyObject() runtime.Object { func (in *EdgeNodeSpec) DeepCopyInto(out *EdgeNodeSpec) { *out = *in out.SSH = in.SSH + out.Host = in.Host out.Edge = in.Edge out.Tunnel = in.Tunnel in.Uplink.DeepCopyInto(&out.Uplink) @@ -176,6 +177,21 @@ func (in *HealthCheckSpec) DeepCopy() *HealthCheckSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *HostSpec) DeepCopyInto(out *HostSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new HostSpec. +func (in *HostSpec) DeepCopy() *HostSpec { + if in == nil { + return nil + } + out := new(HostSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ObjectReference) DeepCopyInto(out *ObjectReference) { *out = *in diff --git a/cmd/tunnelctl/main.go b/cmd/tunnelctl/main.go index e2a7588..49278b0 100644 --- a/cmd/tunnelctl/main.go +++ b/cmd/tunnelctl/main.go @@ -116,7 +116,10 @@ func cmdApply(args []string) error { if err := agentrun.Apply(doc); err != nil { return err } - slog.Info("applied desired state", "interface", doc.WireGuard.Interface.Name, "nftables", doc.Nftables != nil) + slog.Info("applied desired state", + "interface", doc.WireGuard.Interface.Name, + "nftables", doc.Nftables != nil, + "netdev", doc.Netdev != nil) return nil } diff --git a/config/crd/bases/tunnel.achetronic.com_edgenodes.yaml b/config/crd/bases/tunnel.achetronic.com_edgenodes.yaml index 4b18b48..bf18f74 100644 --- a/config/crd/bases/tunnel.achetronic.com_edgenodes.yaml +++ b/config/crd/bases/tunnel.achetronic.com_edgenodes.yaml @@ -102,6 +102,33 @@ spec: type: integer type: object type: object + host: + description: |- + Host tunes the VPS as a machine (kernel and NIC), independent of the proxy. + Optional; sane defaults apply. Placed between SSH and Edge by convention: + host-level OS tuning sits below the proxy configuration. + properties: + disableNicOffloads: + default: false + description: |- + DisableNicOffloads turns GRO/GSO off on the underlay NIC carrying the + WireGuard transport. It defaults false (offloads enabled) because disabling + them degrades throughput for ordinary TCP/QUIC traffic; enable it only for + encapsulated/non-uniform UDP payloads where GRO coalescing can corrupt + datagram boundaries. + type: boolean + kernelMaxSocketBufferBytes: + default: 26214400 + description: |- + KernelMaxSocketBufferBytes is the socket-buffer ceiling. It is threaded to + both the kernel sysctls (net.core.rmem_max/wmem_max) and the + SO_RCVBUF/SO_SNDBUF of Envoy's UDP listeners. UDP does not autotune, so + this single knob raises the effective buffer for high-throughput UDP while + TCP autotunes up to the same ceiling. Defaults to 26214400 (25MB). + format: int64 + minimum: 0 + type: integer + type: object ssh: description: SSH holds the connection details used to enroll and manage the VPS. diff --git a/config/samples/tunnel_v1alpha1_edgenode.yaml b/config/samples/tunnel_v1alpha1_edgenode.yaml index 2338ff0..e373472 100644 --- a/config/samples/tunnel_v1alpha1_edgenode.yaml +++ b/config/samples/tunnel_v1alpha1_edgenode.yaml @@ -51,6 +51,25 @@ spec: # Namespace of the Secret. Defaults to the EdgeNode namespace when omitted. namespace: default + # Host-level performance tuning for the VPS (kernel + NIC), independent of the + # Envoy proxy. Optional; sane defaults apply. + host: + # Socket-buffer ceiling: sets net.core.rmem_max/wmem_max and feeds SO_RCVBUF/ + # SO_SNDBUF on the Envoy UDP listeners (UDP does not autotune like TCP). Raise + # it for high-throughput or tunnel-in-tunnel UDP to avoid packet loss under load. + # Default: 26214400 (25MB). + # Ref: https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt + # Ref: https://www.envoyproxy.io/docs/envoy/latest/api-v3/config/core/v3/socket_option.proto + kernelMaxSocketBufferBytes: 52428800 # Example: 50MB for demanding workloads + + # Turn GRO/GSO off on the underlay NIC (auto-detected via netlink route get). + # Enable it only for encapsulated/non-uniform UDP (tunnel-in-tunnel), where GRO + # coalescing corrupts datagram boundaries and breaks Envoy's L4 datagram + # semantics. WARNING: it degrades throughput for ordinary TCP/QUIC traffic, so + # keep it off unless you hit that case. Default: false (offloads enabled). + # Ref: https://www.kernel.org/doc/html/latest/networking/ethtool-netlink.html + disableNicOffloads: true # Example: enable for encapsulated UDP workloads + # The VPS-side edge proxy (Envoy): how it health checks the uplink replicas and # fails over between them. Optional block; all fields shown with their defaults. edge: diff --git a/deploy/helm/tunnel/crds/tunnel.achetronic.com_edgenodes.yaml b/deploy/helm/tunnel/crds/tunnel.achetronic.com_edgenodes.yaml index 4b18b48..bf18f74 100644 --- a/deploy/helm/tunnel/crds/tunnel.achetronic.com_edgenodes.yaml +++ b/deploy/helm/tunnel/crds/tunnel.achetronic.com_edgenodes.yaml @@ -102,6 +102,33 @@ spec: type: integer type: object type: object + host: + description: |- + Host tunes the VPS as a machine (kernel and NIC), independent of the proxy. + Optional; sane defaults apply. Placed between SSH and Edge by convention: + host-level OS tuning sits below the proxy configuration. + properties: + disableNicOffloads: + default: false + description: |- + DisableNicOffloads turns GRO/GSO off on the underlay NIC carrying the + WireGuard transport. It defaults false (offloads enabled) because disabling + them degrades throughput for ordinary TCP/QUIC traffic; enable it only for + encapsulated/non-uniform UDP payloads where GRO coalescing can corrupt + datagram boundaries. + type: boolean + kernelMaxSocketBufferBytes: + default: 26214400 + description: |- + KernelMaxSocketBufferBytes is the socket-buffer ceiling. It is threaded to + both the kernel sysctls (net.core.rmem_max/wmem_max) and the + SO_RCVBUF/SO_SNDBUF of Envoy's UDP listeners. UDP does not autotune, so + this single knob raises the effective buffer for high-throughput UDP while + TCP autotunes up to the same ceiling. Defaults to 26214400 (25MB). + format: int64 + minimum: 0 + type: integer + type: object ssh: description: SSH holds the connection details used to enroll and manage the VPS. diff --git a/go.mod b/go.mod index 3ff3029..e950b1c 100644 --- a/go.mod +++ b/go.mod @@ -9,10 +9,12 @@ require ( github.com/google/nftables v0.3.0 github.com/onsi/ginkgo/v2 v2.27.2 github.com/onsi/gomega v1.38.2 + github.com/safchain/ethtool v0.7.0 github.com/vishvananda/netlink v1.3.1 golang.org/x/crypto v0.52.0 golang.org/x/sys v0.45.0 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 + gopkg.in/yaml.v3 v3.0.1 k8s.io/api v0.35.0 k8s.io/apimachinery v0.35.0 k8s.io/client-go v0.35.0 @@ -95,7 +97,6 @@ require ( google.golang.org/protobuf v1.36.8 // indirect gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiextensions-apiserver v0.35.0 // indirect k8s.io/apiserver v0.35.0 // indirect k8s.io/component-base v0.35.0 // indirect diff --git a/go.sum b/go.sum index 4ba26ad..709f235 100644 --- a/go.sum +++ b/go.sum @@ -135,6 +135,8 @@ github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlT github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/safchain/ethtool v0.7.0 h1:rlJzfDetsVvT61uz8x1YIcFn12akMfuPulHtZjtb7Is= +github.com/safchain/ethtool v0.7.0/go.mod h1:MenQKEjXdfkjD3mp2QdCk8B/hwvkrlOTm/FD4gTpFxQ= github.com/spf13/cobra v1.10.0 h1:a5/WeUlSDCvV5a45ljW2ZFtV0bTDpkfSAj3uqB6Sc+0= github.com/spf13/cobra v1.10.0/go.mod h1:9dhySC7dnTtEiqzmqfkLj47BslqLCUPMXjG2lj/NgoE= github.com/spf13/pflag v1.0.8/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= diff --git a/internal/agentconfig/agentconfig.go b/internal/agentconfig/agentconfig.go index 37561cc..e0556dc 100644 --- a/internal/agentconfig/agentconfig.go +++ b/internal/agentconfig/agentconfig.go @@ -71,6 +71,9 @@ func (d *Document) Validate() error { if err := validateNftables(d.Nftables); err != nil { return err } + if err := validateNetdev(d.Netdev); err != nil { + return err + } return nil } @@ -152,6 +155,15 @@ func validateNftables(nft *NftablesConfig) error { return nil } +// validateNetdev checks the netdev section. Both a nil section (the NIC is left +// untouched) and a present section are valid: the section carries only a boolean +// toggle, so there is nothing to constrain. It exists to keep the validation +// surface uniform and ready for future fields. +func validateNetdev(nd *NetdevConfig) error { + _ = nd + return nil +} + // validateRule checks every field of the NftablesRule at index i. It returns // the first violation found, naming the offending field. func validateRule(i int, rule NftablesRule) error { diff --git a/internal/agentconfig/agentconfig_test.go b/internal/agentconfig/agentconfig_test.go index ab2a4c6..54e977f 100644 --- a/internal/agentconfig/agentconfig_test.go +++ b/internal/agentconfig/agentconfig_test.go @@ -585,3 +585,74 @@ func TestValidateNilDocument(t *testing.T) { t.Fatal("Validate on nil document should return an error") } } + +// netdevRelayJSON returns a valid relay document carrying a netdev section with +// the given DisableOffloads value, so the netdev field is exercised end to end. +func netdevRelayJSON(disable bool) string { + return `{ + "version": 1, + "wireguard": { + "interface": { + "name": "wg-relay", + "privateKey": "aGVsbG9oZWxsb2hlbGxvaGVsbG9oZWxsb2hlbGxv", + "listenPort": 51821, + "address": "10.200.0.1/24" + }, + "peers": [] + }, + "netdev": { + "disableOffloads": ` + map[bool]string{true: "true", false: "false"}[disable] + ` + } +}` +} + +// TestNetdevSection covers the optional netdev section in all three shapes: +// present with DisableOffloads true, present with false, and absent (nil), plus +// a marshal/parse round-trip that must preserve the section field-for-field. +func TestNetdevSection(t *testing.T) { + // Present, true. + docTrue, err := Parse([]byte(netdevRelayJSON(true))) + if err != nil { + t.Fatalf("parse netdev (true) failed: %v", err) + } + if docTrue.Netdev == nil { + t.Fatal("netdev section should be present") + } + if !docTrue.Netdev.DisableOffloads { + t.Fatal("netdev.disableOffloads should be true") + } + + // Present, false. + docFalse, err := Parse([]byte(netdevRelayJSON(false))) + if err != nil { + t.Fatalf("parse netdev (false) failed: %v", err) + } + if docFalse.Netdev == nil { + t.Fatal("netdev section should be present even when disableOffloads is false") + } + if docFalse.Netdev.DisableOffloads { + t.Fatal("netdev.disableOffloads should be false") + } + + // Absent (nil). + docAbsent, err := Parse([]byte(validRelayJSON)) + if err != nil { + t.Fatalf("parse relay without netdev failed: %v", err) + } + if docAbsent.Netdev != nil { + t.Fatal("netdev section should be nil when omitted") + } + + // Round-trip: Parse -> Marshal -> Parse must preserve the section. + marshaled, err := docTrue.Marshal() + if err != nil { + t.Fatalf("marshal netdev document failed: %v", err) + } + round, err := Parse(marshaled) + if err != nil { + t.Fatalf("re-parse of marshaled netdev document failed: %v", err) + } + if !reflect.DeepEqual(docTrue, round) { + t.Errorf("netdev round-trip mismatch\nwant: %+v\ngot: %+v", docTrue, round) + } +} diff --git a/internal/agentconfig/agentconfig_types.go b/internal/agentconfig/agentconfig_types.go index 5079277..5b78cf9 100644 --- a/internal/agentconfig/agentconfig_types.go +++ b/internal/agentconfig/agentconfig_types.go @@ -21,6 +21,23 @@ type Document struct { WireGuard WireGuardConfig `json:"wireguard"` // Nftables is the optional DNAT ruleset; set only on uplink nodes. Nftables *NftablesConfig `json:"nftables,omitempty"` + // Netdev is the optional host NIC tuning section. A nil section is a no-op + // (the NIC is left untouched); when present it is applied wherever the + // document carries it, independent of the node role. + Netdev *NetdevConfig `json:"netdev,omitempty"` +} + +// NetdevConfig tunes the underlay network interface of the host. tunnelctl +// resolves the underlay interface itself at apply time (the device of the +// default route to a public IP), so this section carries only the desired +// behavior and never an interface name. A nil section means leave the NIC +// untouched; presence with DisableOffloads true turns the offloads off. +type NetdevConfig struct { + // DisableOffloads turns GRO and GSO off on the resolved underlay NIC. It is + // intended for encapsulated/non-uniform UDP payloads where GRO coalescing + // can corrupt datagram boundaries; it defaults off because disabling it + // degrades throughput for ordinary TCP/QUIC traffic. + DisableOffloads bool `json:"disableOffloads"` } // WireGuardConfig describes the WireGuard device of a node: the local interface diff --git a/internal/agentrun/agentrun.go b/internal/agentrun/agentrun.go index d36d1cb..4cbbb10 100644 --- a/internal/agentrun/agentrun.go +++ b/internal/agentrun/agentrun.go @@ -25,6 +25,7 @@ import ( "github.com/fsnotify/fsnotify" "github.com/achetronic/tunnel/internal/agentconfig" + "github.com/achetronic/tunnel/internal/netdev" "github.com/achetronic/tunnel/internal/nftables" "github.com/achetronic/tunnel/internal/wg" ) @@ -49,20 +50,39 @@ const ( // consistent across all three paths. type Loader func() (*agentconfig.Document, error) -// Apply applies the WireGuard config and, when present, the nftables config of -// doc. It is idempotent: re-running with the same document is a no-op. +// Apply applies the WireGuard config and, when present, the nftables and netdev +// configs of doc. It is idempotent: re-running with the same document is a no-op. +// The three appliers are package variables so tests can substitute fakes; the +// defaults are the real native appliers. func Apply(doc *agentconfig.Document) error { - if err := wg.Apply(doc.WireGuard); err != nil { + if err := applyWireGuard(doc.WireGuard); err != nil { return err } if doc.Nftables != nil { - if err := nftables.Apply(*doc.Nftables); err != nil { + if err := applyNftables(*doc.Nftables); err != nil { + return err + } + } + // netdev tunes the host underlay NIC. A nil section is left untouched; a + // present section is applied wherever the document carries it, regardless of + // node role (the relay carries it when the EdgeNode requests it). + if doc.Netdev != nil { + if err := applyNetdev(*doc.Netdev); err != nil { return err } } return nil } +// applyWireGuard, applyNftables and applyNetdev are the native appliers Apply +// dispatches to. They are package variables so tests can replace them without +// touching the kernel; production uses the real implementations. +var ( + applyWireGuard = wg.Apply + applyNftables = nftables.Apply + applyNetdev = netdev.Apply +) + // Readiness decides whether a node is ready from its observed WireGuard state: // the interface must exist, be up, and have at least one peer with a handshake // within handshakeFreshness. diff --git a/internal/agentrun/agentrun_test.go b/internal/agentrun/agentrun_test.go index 67ff3e1..948252b 100644 --- a/internal/agentrun/agentrun_test.go +++ b/internal/agentrun/agentrun_test.go @@ -28,6 +28,50 @@ func testDoc() *agentconfig.Document { } } +// TestApply_DispatchesSections verifies Apply dispatches each native applier: +// WireGuard always, nftables and netdev only when their sections are present. +// The appliers are swapped for fakes so no kernel IO happens; the originals are +// restored after the test. +func TestApply_DispatchesSections(t *testing.T) { + origWG, origNft, origNd := applyWireGuard, applyNftables, applyNetdev + t.Cleanup(func() { + applyWireGuard, applyNftables, applyNetdev = origWG, origNft, origNd + }) + + var wgCalls, nftCalls int + var netdevCalls []agentconfig.NetdevConfig + applyWireGuard = func(agentconfig.WireGuardConfig) error { wgCalls++; return nil } + applyNftables = func(agentconfig.NftablesConfig) error { nftCalls++; return nil } + applyNetdev = func(cfg agentconfig.NetdevConfig) error { + netdevCalls = append(netdevCalls, cfg) + return nil + } + + // WireGuard only: no nftables, no netdev. + if err := Apply(testDoc()); err != nil { + t.Fatalf("apply (wg only) failed: %v", err) + } + if wgCalls != 1 || nftCalls != 0 || len(netdevCalls) != 0 { + t.Fatalf("wg-only dispatch wrong: wg=%d nft=%d netdev=%d", wgCalls, nftCalls, len(netdevCalls)) + } + + // With a netdev section present, netdev.Apply must be dispatched with it. + doc := testDoc() + doc.Netdev = &agentconfig.NetdevConfig{DisableOffloads: true} + if err := Apply(doc); err != nil { + t.Fatalf("apply (with netdev) failed: %v", err) + } + if len(netdevCalls) != 1 || !netdevCalls[0].DisableOffloads { + t.Fatalf("netdev section not dispatched as expected: %+v", netdevCalls) + } + + // A netdev applier error must propagate. + applyNetdev = func(agentconfig.NetdevConfig) error { return errors.New("boom") } + if err := Apply(doc); err == nil { + t.Fatal("expected netdev apply error to propagate") + } +} + // The retry loop must keep attempting with exponentially increasing sleeps // until an apply succeeds, then stop. func TestRetryInitialApply_RetriesUntilSuccess(t *testing.T) { diff --git a/internal/netdev/netdev.go b/internal/netdev/netdev.go new file mode 100644 index 0000000..e0be204 --- /dev/null +++ b/internal/netdev/netdev.go @@ -0,0 +1,139 @@ +// SPDX-FileCopyrightText: 2026 Alby Hernández (Achetronic) +// SPDX-License-Identifier: Apache-2.0 + +// Package netdev applies host network-interface tuning natively, without +// shelling out to the ethtool CLI. Today it toggles the GRO/GSO offloads of the +// underlay NIC that carries the WireGuard transport. The underlay interface is +// resolved at apply time from the route to a public IP (not hardcoded, never the +// wg tunnel), so the same document applies on any host. The kernel IO is kept +// behind seams (resolve + feature controller) so the decision logic is unit +// tested without root. +package netdev + +import ( + "fmt" + "net" + + "github.com/safchain/ethtool" + "github.com/vishvananda/netlink" + + "github.com/achetronic/tunnel/internal/agentconfig" +) + +// offloadFeatureKeys are the kernel netdev feature names this package disables. +// They are the strings the kernel exposes in netdev_features_strings: +// "rx-gro" is NETIF_F_GRO_BIT (Generic Receive Offload) and +// "tx-generic-segmentation" is NETIF_F_GSO_BIT (Generic Segmentation Offload). +// Using the kernel feature names (resolved against the device's GSTRINGS by the +// ethtool library) is robust across kernel versions, unlike raw feature bits. +var offloadFeatureKeys = []string{"rx-gro", "tx-generic-segmentation"} + +// underlayProbeIP is a public address used only to resolve which interface the +// host routes outbound traffic through. No packet is sent to it; the kernel's +// route lookup returns the outbound device, which is the underlay NIC carrying +// the WireGuard transport (never the wg tunnel, whose routes are tunnel /32s). +const underlayProbeIP = "1.1.1.1" + +// relayInterfaceName is the WireGuard tunnel device on the relay. The underlay +// resolver guards against ever selecting it: tuning the tunnel device is never +// the intent, only the physical NIC beneath it. +const relayInterfaceName = "wg-relay" + +// featureController is the IO seam over the host's offload features. Production +// uses safchain/ethtool over the SIOCETHTOOL ioctl; tests inject a fake. +type featureController interface { + // Features returns the current offload features keyed by kernel name. + Features(iface string) (map[string]bool, error) + // Change requests the given features be set to the given on/off states. + Change(iface string, config map[string]bool) error +} + +// interfaceResolver is the IO seam that resolves the underlay interface name. +// Production resolves it via a netlink route lookup; tests inject a fake. +type interfaceResolver func() (string, error) + +// Apply brings the host NIC to the desired state described by cfg. A section +// with DisableOffloads false is a no-op (offloads are never re-enabled here: +// presence-disables, absence/false leaves them untouched). When DisableOffloads +// is true it opens an ethtool handle, resolves the underlay interface and turns +// GRO/GSO off idempotently. Re-running with the same input is a no-op. +func Apply(cfg agentconfig.NetdevConfig) error { + if !cfg.DisableOffloads { + return nil + } + handle, err := ethtool.NewEthtool() + if err != nil { + return fmt.Errorf("netdev: open ethtool: %w", err) + } + defer handle.Close() + return applyOffloads(resolveUnderlayInterface, handle, cfg) +} + +// applyOffloads is the pure decision core behind Apply: it resolves the +// interface, reads its current offload features and only issues a Change for the +// features that are present and still enabled, so it is idempotent and never +// fails on a driver that does not expose one of the keys. It is split out so the +// logic is exercised in unit tests with injected seams (no root, no NIC). +func applyOffloads(resolve interfaceResolver, ctrl featureController, cfg agentconfig.NetdevConfig) error { + if !cfg.DisableOffloads { + return nil + } + iface, err := resolve() + if err != nil { + return fmt.Errorf("netdev: resolve underlay interface: %w", err) + } + + current, err := ctrl.Features(iface) + if err != nil { + return fmt.Errorf("netdev: read features of %q: %w", iface, err) + } + + desired := make(map[string]bool, len(offloadFeatureKeys)) + for _, key := range offloadFeatureKeys { + // A feature the driver does not report cannot be disabled; skip it + // rather than asking the kernel to change an unknown feature. + if enabled, ok := current[key]; ok && enabled { + desired[key] = false + } + } + if len(desired) == 0 { + // Already off (or unsupported): nothing to do, stay idempotent. + return nil + } + + if err := ctrl.Change(iface, desired); err != nil { + return fmt.Errorf("netdev: disable offloads on %q: %w", iface, err) + } + return nil +} + +// resolveUnderlayInterface returns the name of the interface the host uses to +// reach the public internet, by asking the kernel for the route to a public IP +// and mapping the outbound link index to a name. It deliberately skips the wg +// tunnel device so the offload toggle always lands on the physical NIC beneath +// it. No traffic is sent; only the routing table is consulted. +func resolveUnderlayInterface() (string, error) { + ip := net.ParseIP(underlayProbeIP) + if ip == nil { + return "", fmt.Errorf("invalid probe IP %q", underlayProbeIP) + } + routes, err := netlink.RouteGet(ip) + if err != nil { + return "", fmt.Errorf("route get to %s: %w", underlayProbeIP, err) + } + for _, r := range routes { + if r.LinkIndex <= 0 { + continue + } + link, err := netlink.LinkByIndex(r.LinkIndex) + if err != nil { + return "", fmt.Errorf("look up link index %d: %w", r.LinkIndex, err) + } + name := link.Attrs().Name + if name == "" || name == relayInterfaceName { + continue + } + return name, nil + } + return "", fmt.Errorf("no outbound underlay interface found for %s", underlayProbeIP) +} diff --git a/internal/netdev/netdev_kernel_test.go b/internal/netdev/netdev_kernel_test.go new file mode 100644 index 0000000..82d6af5 --- /dev/null +++ b/internal/netdev/netdev_kernel_test.go @@ -0,0 +1,97 @@ +// SPDX-FileCopyrightText: 2026 Alby Hernández (Achetronic) +// SPDX-License-Identifier: Apache-2.0 + +package netdev + +import ( + "errors" + "fmt" + "os" + "testing" + + "github.com/safchain/ethtool" + "github.com/vishvananda/netlink" + + "github.com/achetronic/tunnel/internal/agentconfig" +) + +// TestApplyOffloads_Kernel exercises the real ethtool IO path against a throwaway +// dummy interface: it opens the SIOCETHTOOL handle, reads the device offload +// features and best-effort disables GRO/GSO through applyOffloads with the dummy +// pinned as the underlay. +// +// It needs CAP_NET_ADMIN (run as root: `sudo go test ./internal/netdev/`). +// It self-skips with an explicit "SKIPPED, NOT TESTED" message when unprivileged, +// and also when the kernel/dummy driver does not expose the offload features +// (dummy interfaces frequently report a fixed feature set that cannot be +// toggled), so the default `go test ./...` and unprivileged CI stay green while a +// sudo CI step provides the coverage. +func TestApplyOffloads_Kernel(t *testing.T) { + handle, err := ethtool.NewEthtool() + if err != nil { + if errors.Is(err, os.ErrPermission) { + t.Skipf("SKIPPED, NOT TESTED: the ethtool offload path was NOT exercised "+ + "because opening the SIOCETHTOOL socket requires root (CAP_NET_ADMIN). "+ + "Run `sudo go test ./internal/netdev/ -run TestApplyOffloads_Kernel -v` "+ + "to get this coverage (CI runners with passwordless sudo can do it as a step): %v", err) + } + t.Skipf("SKIPPED, NOT TESTED: ethtool is unavailable in this environment: %v", err) + } + defer handle.Close() + + linkName := fmt.Sprintf("tnlnd%d", os.Getpid()%10000) + link := &netlink.Dummy{LinkAttrs: netlink.LinkAttrs{Name: linkName}} + if err := netlink.LinkAdd(link); err != nil { + if errors.Is(err, os.ErrPermission) { + t.Skipf("SKIPPED, NOT TESTED: the offload toggle was NOT exercised because "+ + "creating a network interface requires root (CAP_NET_ADMIN). "+ + "Run `sudo go test ./internal/netdev/ -run TestApplyOffloads_Kernel -v` "+ + "to get this coverage: %v", err) + } + t.Fatalf("create dummy link: %v", err) + } + t.Logf("root privileges found: exercising real ethtool offload toggling "+ + "against the throwaway dummy interface %q (deleted on cleanup)", linkName) + t.Cleanup(func() { _ = netlink.LinkDel(link) }) + + features, err := handle.Features(linkName) + if err != nil { + t.Skipf("SKIPPED, NOT TESTED: the dummy interface %q does not support reading "+ + "ethtool features in this environment, so the offload toggle could not be "+ + "exercised (this is expected for dummy drivers): %v", linkName, err) + } + + togglable := false + for _, key := range offloadFeatureKeys { + if _, ok := features[key]; ok { + togglable = true + break + } + } + if !togglable { + t.Skipf("SKIPPED, NOT TESTED: the dummy interface %q exposes none of %v, so the "+ + "offload toggle could not be exercised (dummy drivers report a fixed feature "+ + "set); run this on a host with a real NIC for full coverage", linkName, offloadFeatureKeys) + } + + resolve := staticResolver(linkName) + if err := applyOffloads(resolve, handle, agentconfig.NetdevConfig{DisableOffloads: true}); err != nil { + t.Skipf("SKIPPED, NOT TESTED: the dummy interface %q does not allow toggling the "+ + "offload features (drivers may pin them), so the change could not be applied: %v", linkName, err) + } + + // Idempotency: re-running with the features now off must not error. + if err := applyOffloads(resolve, handle, agentconfig.NetdevConfig{DisableOffloads: true}); err != nil { + t.Fatalf("idempotent re-apply failed: %v", err) + } + + after, err := handle.Features(linkName) + if err != nil { + t.Fatalf("re-read features: %v", err) + } + for _, key := range offloadFeatureKeys { + if enabled, ok := after[key]; ok && enabled { + t.Errorf("feature %q is still enabled after disabling offloads", key) + } + } +} diff --git a/internal/netdev/netdev_test.go b/internal/netdev/netdev_test.go new file mode 100644 index 0000000..6faab74 --- /dev/null +++ b/internal/netdev/netdev_test.go @@ -0,0 +1,135 @@ +// SPDX-FileCopyrightText: 2026 Alby Hernández (Achetronic) +// SPDX-License-Identifier: Apache-2.0 + +package netdev + +import ( + "fmt" + "maps" + "testing" + + "github.com/achetronic/tunnel/internal/agentconfig" +) + +// fakeController is an in-memory featureController for unit tests. It records the +// Change calls so a test can assert exactly which features were toggled. +type fakeController struct { + features map[string]bool + featuresErr error + changeErr error + changes []map[string]bool +} + +// Features returns the canned feature map (or the canned error). +func (f *fakeController) Features(string) (map[string]bool, error) { + if f.featuresErr != nil { + return nil, f.featuresErr + } + out := make(map[string]bool, len(f.features)) + maps.Copy(out, f.features) + return out, nil +} + +// Change records the requested change (or returns the canned error). +func (f *fakeController) Change(_ string, config map[string]bool) error { + if f.changeErr != nil { + return f.changeErr + } + f.changes = append(f.changes, config) + return nil +} + +// staticResolver returns a fixed interface name, standing in for the netlink +// route lookup so unit tests need no real NIC. +func staticResolver(name string) interfaceResolver { + return func() (string, error) { return name, nil } +} + +// TestApplyOffloads covers the decision core across every meaningful shape: +// disabled section is a no-op, both offloads on get turned off, already-off and +// unsupported features are left alone, and read/change errors are wrapped. +func TestApplyOffloads(t *testing.T) { + t.Run("disable offloads false is a no-op", func(t *testing.T) { + ctrl := &fakeController{features: map[string]bool{"rx-gro": true}} + if err := applyOffloads(staticResolver("eth0"), ctrl, agentconfig.NetdevConfig{DisableOffloads: false}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(ctrl.changes) != 0 { + t.Fatalf("no change expected when DisableOffloads is false, got %v", ctrl.changes) + } + }) + + t.Run("both offloads on are turned off", func(t *testing.T) { + ctrl := &fakeController{features: map[string]bool{ + "rx-gro": true, + "tx-generic-segmentation": true, + "tx-checksum-ip-generic": true, + }} + if err := applyOffloads(staticResolver("eth0"), ctrl, agentconfig.NetdevConfig{DisableOffloads: true}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(ctrl.changes) != 1 { + t.Fatalf("expected exactly one Change call, got %d", len(ctrl.changes)) + } + got := ctrl.changes[0] + if len(got) != 2 || got["rx-gro"] != false || got["tx-generic-segmentation"] != false { + t.Fatalf("expected GRO+GSO disabled only, got %v", got) + } + }) + + t.Run("already off is idempotent no-op", func(t *testing.T) { + ctrl := &fakeController{features: map[string]bool{ + "rx-gro": false, + "tx-generic-segmentation": false, + }} + if err := applyOffloads(staticResolver("eth0"), ctrl, agentconfig.NetdevConfig{DisableOffloads: true}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(ctrl.changes) != 0 { + t.Fatalf("no change expected when offloads already off, got %v", ctrl.changes) + } + }) + + t.Run("unsupported feature is skipped", func(t *testing.T) { + // The driver only reports GSO; GRO is absent and must not be requested. + ctrl := &fakeController{features: map[string]bool{"tx-generic-segmentation": true}} + if err := applyOffloads(staticResolver("eth0"), ctrl, agentconfig.NetdevConfig{DisableOffloads: true}); err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(ctrl.changes) != 1 { + t.Fatalf("expected one Change call, got %d", len(ctrl.changes)) + } + got := ctrl.changes[0] + if len(got) != 1 || got["tx-generic-segmentation"] != false { + t.Fatalf("expected only GSO disabled, got %v", got) + } + if _, ok := got["rx-gro"]; ok { + t.Fatal("unsupported rx-gro must not be requested") + } + }) + + t.Run("read features error is wrapped", func(t *testing.T) { + ctrl := &fakeController{featuresErr: fmt.Errorf("boom")} + err := applyOffloads(staticResolver("eth0"), ctrl, agentconfig.NetdevConfig{DisableOffloads: true}) + if err == nil { + t.Fatal("expected an error when reading features fails") + } + }) + + t.Run("change error is wrapped", func(t *testing.T) { + ctrl := &fakeController{features: map[string]bool{"rx-gro": true}, changeErr: fmt.Errorf("denied")} + err := applyOffloads(staticResolver("eth0"), ctrl, agentconfig.NetdevConfig{DisableOffloads: true}) + if err == nil { + t.Fatal("expected an error when Change fails") + } + }) + + t.Run("resolver error is wrapped", func(t *testing.T) { + failing := func() (string, error) { return "", fmt.Errorf("no route") } + ctrl := &fakeController{features: map[string]bool{"rx-gro": true}} + err := applyOffloads(failing, ctrl, agentconfig.NetdevConfig{DisableOffloads: true}) + if err == nil { + t.Fatal("expected an error when interface resolution fails") + } + }) +} diff --git a/internal/planner/plan.go b/internal/planner/plan.go index cd6c6bb..82425b5 100644 --- a/internal/planner/plan.go +++ b/internal/planner/plan.go @@ -83,12 +83,22 @@ func BuildPlan( keepalive = 25 } + // Host tuning: default the socket-buffer ceiling and read the NIC offload + // toggle. The buffer is threaded into the Envoy UDP socket_options and the + // sysctl drop-in (via the Plan); the offload toggle becomes a netdev section + // in the relay document only when requested. + bufferBytes := node.Spec.Host.KernelMaxSocketBufferBytes + if bufferBytes <= 0 { + bufferBytes = defaultKernelMaxSocketBufferBytes + } + disableOffloads := node.Spec.Host.DisableNicOffloads + wgPeers, err := buildRelayPeers(ipCalc, uplinkKeys, replicas) if err != nil { return nil, err } - relayDocument, err := buildRelayDocument(vpsPrivKey, fmt.Sprintf("%s/%d", relayIP, mask), listenPort, mtu, wgPeers) + relayDocument, err := buildRelayDocument(vpsPrivKey, fmt.Sprintf("%s/%d", relayIP, mask), listenPort, mtu, wgPeers, disableOffloads) if err != nil { return nil, fmt.Errorf("render relay document: %w", err) } @@ -110,14 +120,16 @@ func BuildPlan( } envoyLDS, err := render.RenderEnvoyLDS(render.EnvoyConfig{ - Listeners: envoyListeners, + Listeners: envoyListeners, + UDPSocketBufferBytes: bufferBytes, }) if err != nil { return nil, fmt.Errorf("render envoy LDS: %w", err) } envoyCDS, err := render.RenderEnvoyCDS(render.EnvoyConfig{ - Listeners: envoyListeners, + Listeners: envoyListeners, + UDPSocketBufferBytes: bufferBytes, }) if err != nil { return nil, fmt.Errorf("render envoy CDS: %w", err) @@ -140,20 +152,26 @@ func BuildPlan( planHash := hashBytes([]byte(relayDocHash + uplinkDocHash + ldsHash + cdsHash)) return &Plan{ - EnvoyLDS: envoyLDS, - EnvoyCDS: envoyCDS, - EnvoyLDSHash: ldsHash, - EnvoyCDSHash: cdsHash, - RelayDocument: relayDocument, - RelayDocumentHash: relayDocHash, - UplinkDocument: uplinkDocument, - UplinkDocumentHash: uplinkDocHash, - PlanHash: planHash, - RelayIP: relayIP, - TLSMaterials: tlsMaterials, + EnvoyLDS: envoyLDS, + EnvoyCDS: envoyCDS, + EnvoyLDSHash: ldsHash, + EnvoyCDSHash: cdsHash, + RelayDocument: relayDocument, + RelayDocumentHash: relayDocHash, + UplinkDocument: uplinkDocument, + UplinkDocumentHash: uplinkDocHash, + PlanHash: planHash, + RelayIP: relayIP, + TLSMaterials: tlsMaterials, + KernelMaxSocketBufferBytes: bufferBytes, }, nil } +// defaultKernelMaxSocketBufferBytes is the socket-buffer ceiling used when the +// EdgeNode leaves spec.host.kernelMaxSocketBufferBytes unset or non-positive. It +// matches the kubebuilder default on v1alpha1.HostSpec (25MB). +const defaultKernelMaxSocketBufferBytes int64 = 26214400 + // relayInterface is the WireGuard interface name on the VPS relay. const relayInterface = "wg-relay" @@ -168,8 +186,10 @@ const metricsPort = 40600 // buildRelayDocument renders the tunnelctl desired-state document for the VPS // relay. tunnelctl applies it natively via netlink. The relay has no nftables // section; its peers are the uplink replicas, reached only when they dial in, so -// they carry no endpoint or keepalive. -func buildRelayDocument(privateKey, interfaceIP string, listenPort, mtu int32, peers []agentconfig.WireGuardPeer) ([]byte, error) { +// they carry no endpoint or keepalive. When disableOffloads is true the document +// carries a netdev section that turns the underlay NIC's GRO/GSO off; otherwise +// the section is left nil so the NIC is untouched. +func buildRelayDocument(privateKey, interfaceIP string, listenPort, mtu int32, peers []agentconfig.WireGuardPeer, disableOffloads bool) ([]byte, error) { doc := agentconfig.Document{ Version: agentconfig.CurrentVersion, WireGuard: agentconfig.WireGuardConfig{ @@ -183,6 +203,9 @@ func buildRelayDocument(privateKey, interfaceIP string, listenPort, mtu int32, p Peers: peers, }, } + if disableOffloads { + doc.Netdev = &agentconfig.NetdevConfig{DisableOffloads: true} + } if err := doc.Validate(); err != nil { return nil, fmt.Errorf("invalid relay document: %w", err) } diff --git a/internal/planner/plan_test.go b/internal/planner/plan_test.go index a7f6684..03125d8 100644 --- a/internal/planner/plan_test.go +++ b/internal/planner/plan_test.go @@ -590,6 +590,109 @@ func TestBuildPlan_RelayDocument(t *testing.T) { t.Errorf("relay peer[%d] keepalive = %d, want 0", i, p.PersistentKeepalive) } } + // With host tuning unset, the relay document carries no netdev section. + if doc.Netdev != nil { + t.Error("relay document must not carry a netdev section when offloads are not disabled") + } +} + +// TestBuildPlan_HostTuning verifies the host-level performance tuning is threaded +// through the plan: the socket-buffer ceiling defaults when unset and reaches the +// Plan and the Envoy UDP socket_options, and the relay document carries a netdev +// section only when the EdgeNode requests disabling NIC offloads. +func TestBuildPlan_HostTuning(t *testing.T) { + keys := map[int32]string{0: "pub0", 1: "pub1"} + udpBinding := []v1alpha1.PortBinding{ + { + Spec: v1alpha1.PortBindingSpec{ + Bindings: []v1alpha1.PortBindingDefinition{ + { + Name: "dns", + Protocol: "UDP", + ListenPort: 53, + Target: v1alpha1.BindingTarget{Address: "10.96.1.2", Port: 53}, + }, + }, + }, + }, + } + + t.Run("defaults when host unset", func(t *testing.T) { + node := makeNode() + node.Spec.Address = testVPSAddress + plan, err := BuildPlan(node, udpBinding, mockResolver{}, "priv", "pub", keys) + if err != nil { + t.Fatal(err) + } + if plan.KernelMaxSocketBufferBytes != defaultKernelMaxSocketBufferBytes { + t.Errorf("buffer = %d, want default %d", plan.KernelMaxSocketBufferBytes, defaultKernelMaxSocketBufferBytes) + } + // The default buffer is non-zero, so UDP listeners get socket_options. + if !strings.Contains(string(plan.EnvoyLDS), "socket_options") { + t.Errorf("LDS should contain UDP socket_options with the default buffer;\n%s", plan.EnvoyLDS) + } + if !strings.Contains(string(plan.EnvoyLDS), "int_value: 26214400") { + t.Errorf("LDS should thread the default buffer into socket_options;\n%s", plan.EnvoyLDS) + } + doc, err := agentconfig.Parse(plan.RelayDocument) + if err != nil { + t.Fatalf("relay document does not validate: %v", err) + } + if doc.Netdev != nil { + t.Error("netdev section must be absent when DisableNicOffloads is false") + } + }) + + t.Run("custom buffer is threaded", func(t *testing.T) { + node := makeNode() + node.Spec.Address = testVPSAddress + node.Spec.Host.KernelMaxSocketBufferBytes = 8388608 + plan, err := BuildPlan(node, udpBinding, mockResolver{}, "priv", "pub", keys) + if err != nil { + t.Fatal(err) + } + if plan.KernelMaxSocketBufferBytes != 8388608 { + t.Errorf("buffer = %d, want 8388608", plan.KernelMaxSocketBufferBytes) + } + if !strings.Contains(string(plan.EnvoyLDS), "int_value: 8388608") { + t.Errorf("LDS should thread the custom buffer into socket_options;\n%s", plan.EnvoyLDS) + } + }) + + t.Run("netdev section present when offloads disabled", func(t *testing.T) { + node := makeNode() + node.Spec.Address = testVPSAddress + node.Spec.Host.DisableNicOffloads = true + plan, err := BuildPlan(node, udpBinding, mockResolver{}, "priv", "pub", keys) + if err != nil { + t.Fatal(err) + } + doc, err := agentconfig.Parse(plan.RelayDocument) + if err != nil { + t.Fatalf("relay document does not validate: %v", err) + } + if doc.Netdev == nil || !doc.Netdev.DisableOffloads { + t.Fatalf("relay document must carry netdev.disableOffloads=true; got %+v", doc.Netdev) + } + }) + + t.Run("hashes stable for identical host tuning", func(t *testing.T) { + node := makeNode() + node.Spec.Address = testVPSAddress + node.Spec.Host.DisableNicOffloads = true + node.Spec.Host.KernelMaxSocketBufferBytes = 12345678 + p1, err := BuildPlan(node, udpBinding, mockResolver{}, "priv", "pub", keys) + if err != nil { + t.Fatal(err) + } + p2, err := BuildPlan(node, udpBinding, mockResolver{}, "priv", "pub", keys) + if err != nil { + t.Fatal(err) + } + if p1.PlanHash != p2.PlanHash || p1.RelayDocumentHash != p2.RelayDocumentHash { + t.Error("plan/relay hashes must be stable for identical host tuning input") + } + }) } // TestBuildPlan_UplinkDocument verifies the shared uplink tunnelctl document: diff --git a/internal/planner/plan_types.go b/internal/planner/plan_types.go index b7c03f8..0e3f04c 100644 --- a/internal/planner/plan_types.go +++ b/internal/planner/plan_types.go @@ -113,6 +113,14 @@ type Plan struct { // SSH. Like EnvoyVersion it is an install-time input set by the controller // from the manager's --tunnelctl-dir flag, so it does not feed any of the hashes. TunnelctlDir string + + // KernelMaxSocketBufferBytes is the host socket-buffer ceiling (defaulted by + // the planner). The provision layer writes it into the sysctl drop-in + // (net.core.rmem_max/wmem_max) on the VPS. It is already reflected in the + // rendered Envoy LDS (UDP socket_options) and, when the EdgeNode requests it, + // in the relay document's netdev section, so it feeds the hashes through those + // artifacts and is not folded into PlanHash separately. + KernelMaxSocketBufferBytes int64 } // hashBytes returns the hex-encoded SHA-256 digest of b. diff --git a/internal/provision/enroll.go b/internal/provision/enroll.go index 4a992d2..0d0e046 100644 --- a/internal/provision/enroll.go +++ b/internal/provision/enroll.go @@ -111,7 +111,7 @@ func Enroll(ctx context.Context, exec sshexec.Executor, plan *planner.Plan, tlsF if err := writeEnvoyService(ctx, exec); err != nil { return false, err } - if err := enableIPForwarding(ctx, exec); err != nil { + if err := writeHostSysctls(ctx, exec, plan.KernelMaxSocketBufferBytes); err != nil { return false, err } if err := applyRelayDocument(ctx, exec, plan, state); err != nil { @@ -261,16 +261,19 @@ func installEnvoyBinary(ctx context.Context, exec sshexec.Executor, version stri } // writeEnvoyService installs the Envoy systemd unit and a boot-time oneshot -// that reapplies the WireGuard relay, then reloads systemd and enables both so -// they come back after a VPS reboot. The wg-relay interface is created natively -// by tunnelctl and is not kernel-persistent, so without a boot unit a reboot -// leaves Envoy unable to bind its admin to the tunnel IP and unable to reach the -// uplinks until the next operator reconcile. The wg-relay oneshot runs the same -// tunnelctl already on the host (no resident daemon), and the Envoy unit depends -// on it so WireGuard is up before Envoy starts. +// (tunnel-boot.service) that reapplies the whole node desired-state document, +// then reloads systemd and enables both so they come back after a VPS reboot. +// The wg-relay interface is created natively by tunnelctl and is not +// kernel-persistent, so without a boot unit a reboot leaves Envoy unable to bind +// its admin to the tunnel IP and unable to reach the uplinks until the next +// operator reconcile. The tunnel-boot oneshot runs the same tunnelctl already on +// the host (no resident daemon), reapplying the relay document (WireGuard plus +// any netdev tuning), and the Envoy unit depends on it so the relay is up before +// Envoy starts. The systemd unit is named tunnel-boot.service; the WireGuard +// interface it brings up is still wg-relay. func writeEnvoyService(ctx context.Context, exec sshexec.Executor) error { - wgRelayService := `[Unit] -Description=Tunnel WireGuard relay + bootService := `[Unit] +Description=Tunnel node boot reconcile After=network.target [Service] @@ -283,8 +286,8 @@ WantedBy=multi-user.target ` envoyService := `[Unit] Description=Envoy Proxy -After=network.target wg-relay.service -Requires=wg-relay.service +After=network.target tunnel-boot.service +Requires=tunnel-boot.service [Service] ExecStart=` + envoyBinPath + ` -c /etc/envoy/envoy.yaml @@ -298,8 +301,8 @@ WantedBy=multi-user.target if _, err := exec.Run(ctx, "mkdir -p /etc/envoy"); err != nil { return fmt.Errorf("failed to create envoy dir: %w", err) } - if err := exec.Put(ctx, "/etc/systemd/system/wg-relay.service", []byte(wgRelayService)); err != nil { - return fmt.Errorf("failed to write wg-relay.service: %w", err) + if err := exec.Put(ctx, "/etc/systemd/system/tunnel-boot.service", []byte(bootService)); err != nil { + return fmt.Errorf("failed to write tunnel-boot.service: %w", err) } if err := exec.Put(ctx, "/etc/systemd/system/envoy.service", []byte(envoyService)); err != nil { return fmt.Errorf("failed to write envoy.service: %w", err) @@ -307,20 +310,51 @@ WantedBy=multi-user.target if _, err := exec.Run(ctx, "systemctl daemon-reload"); err != nil { return fmt.Errorf("failed to reload systemd: %w", err) } - // Enable wg-relay so the relay is reapplied on boot. Envoy is enabled - // alongside its start in ensureEnvoyRunning; enabling wg-relay here keeps + // Enable tunnel-boot so the relay is reapplied on boot. Envoy is enabled + // alongside its start in ensureEnvoyRunning; enabling tunnel-boot here keeps // the boot-survival contract in one place. - if _, err := exec.Run(ctx, "systemctl enable wg-relay.service"); err != nil { - return fmt.Errorf("failed to enable wg-relay.service: %w", err) + if _, err := exec.Run(ctx, "systemctl enable tunnel-boot.service"); err != nil { + return fmt.Errorf("failed to enable tunnel-boot.service: %w", err) } return nil } -// enableIPForwarding turns on IPv4 forwarding so the relay can route traffic -// between peers, persisting it through a sysctl drop-in. -func enableIPForwarding(ctx context.Context, exec sshexec.Executor) error { - if _, err := exec.Run(ctx, "sysctl -w net.ipv4.ip_forward=1 && echo 'net.ipv4.ip_forward=1' > /etc/sysctl.d/99-tunnel.conf"); err != nil { - return fmt.Errorf("failed to set sysctl: %w", err) +// sysctlDropInPath is the persisted sysctl drop-in the operator owns on the VPS. +const sysctlDropInPath = "/etc/sysctl.d/99-tunnel.conf" + +// defaultKernelMaxSocketBufferBytes is the socket-buffer ceiling used when the +// plan carries a non-positive value, mirroring the planner/CRD default (25MB). +// It keeps the sysctl drop-in valid even if a caller builds a Plan directly. +const defaultKernelMaxSocketBufferBytes int64 = 26214400 + +// writeHostSysctls installs the operator's sysctl drop-in on the VPS in a single +// overwrite (never an append, so re-applying never duplicates lines) and applies +// it with "sysctl -p". It enables IPv4 forwarding so the relay routes between +// WireGuard peers, and raises the kernel socket-buffer ceiling +// (net.core.rmem_max/wmem_max) to the configured value so high-throughput UDP +// listeners can use larger SO_RCVBUF/SO_SNDBUF (UDP does not autotune). +func writeHostSysctls(ctx context.Context, exec sshexec.Executor, bufferBytes int64) error { + if bufferBytes <= 0 { + bufferBytes = defaultKernelMaxSocketBufferBytes + } + // Managed drop-in: a single overwrite keeps it idempotent across reconciles. + // Docs: kernel ip-sysctl (ip_forward) and admin-guide/sysctl/net (core buffers). + content := fmt.Sprintf(`# Managed by Tunnel. Do not edit; this file is overwritten on every reconcile. +# IPv4 forwarding lets the relay route traffic between WireGuard peers. +# https://www.kernel.org/doc/Documentation/networking/ip-sysctl.txt +net.ipv4.ip_forward=1 +# Socket-buffer ceiling for high-throughput UDP listeners. UDP does not autotune, +# so Envoy sets SO_RCVBUF/SO_SNDBUF up to this maximum; TCP autotunes below it. +# https://www.kernel.org/doc/Documentation/admin-guide/sysctl/net.rst +net.core.rmem_max=%d +net.core.wmem_max=%d +`, bufferBytes, bufferBytes) + + if err := exec.Put(ctx, sysctlDropInPath, []byte(content)); err != nil { + return fmt.Errorf("failed to write sysctl drop-in: %w", err) + } + if _, err := exec.Run(ctx, "sysctl -p "+sysctlDropInPath); err != nil { + return fmt.Errorf("failed to apply sysctl drop-in: %w", err) } return nil } diff --git a/internal/provision/enroll_test.go b/internal/provision/enroll_test.go index 6ff2310..cd75b72 100644 --- a/internal/provision/enroll_test.go +++ b/internal/provision/enroll_test.go @@ -285,9 +285,9 @@ func TestTeardown(t *testing.T) { "ip link del wg-relay", "rm -f " + tunnelctlBinPath, "systemctl disable --now envoy", - "systemctl disable --now wg-relay.service", + "systemctl disable --now tunnel-boot.service", "rm -f /etc/systemd/system/envoy.service", - "rm -f /etc/systemd/system/wg-relay.service", + "rm -f /etc/systemd/system/tunnel-boot.service", "systemctl daemon-reload", "rm -rf /etc/envoy", "rm -f /etc/sysctl.d/99-tunnel.conf", @@ -905,10 +905,10 @@ func TestEnroll_CorruptedStateReEnrolls(t *testing.T) { } // TestEnroll_RebootSurvivalAndConfigOrder verifies the enroll robustness fixes: -// a wg-relay boot oneshot is installed and enabled, the Envoy unit depends on -// it, the stale-temp sweep covers /etc/systemd/system, and CDS is written -// before LDS so a write interruption never leaves a listener referencing an -// absent cluster. +// a tunnel-boot oneshot is installed and enabled, the Envoy unit depends on +// it, the stale-temp sweep covers /etc/systemd/system, CDS is written before LDS +// so a write interruption never leaves a listener referencing an absent cluster, +// and the host sysctl drop-in carries ip_forward plus the socket-buffer ceiling. func TestEnroll_RebootSurvivalAndConfigOrder(t *testing.T) { fake := sshexec.NewFakeExecutor() fake.RunFunc = func(ctx context.Context, cmd string) (string, error) { @@ -940,15 +940,16 @@ func TestEnroll_RebootSurvivalAndConfigOrder(t *testing.T) { } plan := &planner.Plan{ - RelayDocument: []byte(`{"version":1}`), - EnvoyLDS: []byte("lds"), - EnvoyCDS: []byte("cds"), - RelayDocumentHash: "h1", - TunnelctlDir: tunnelctlFixtureDir(t), - EnvoyVersion: "1.30.1", - EnvoyLDSHash: "h2", - EnvoyCDSHash: "h3", - RelayIP: "10.200.0.1", + RelayDocument: []byte(`{"version":1}`), + EnvoyLDS: []byte("lds"), + EnvoyCDS: []byte("cds"), + RelayDocumentHash: "h1", + TunnelctlDir: tunnelctlFixtureDir(t), + EnvoyVersion: "1.30.1", + EnvoyLDSHash: "h2", + EnvoyCDSHash: "h3", + RelayIP: "10.200.0.1", + KernelMaxSocketBufferBytes: 8388608, } if _, err := Enroll(context.Background(), fake, plan, nil); err != nil { @@ -957,22 +958,39 @@ func TestEnroll_RebootSurvivalAndConfigOrder(t *testing.T) { joined := strings.Join(fake.Runs, "\n") - // FIX reboot: the wg-relay boot oneshot must be installed with a tunnelctl - // apply ExecStart for the relay document, the Envoy unit must depend on it, - // and wg-relay must be enabled so it runs on boot. - wgUnit := string(fake.Files["/etc/systemd/system/wg-relay.service"]) - if !strings.Contains(wgUnit, "ExecStart="+tunnelctlBinPath+" apply --config "+relayDocumentPath) { - t.Fatalf("wg-relay.service missing tunnelctl apply ExecStart; got:\n%s", wgUnit) + // The tunnel-boot oneshot must be installed with a tunnelctl apply ExecStart + // for the relay document, the Envoy unit must depend on it, and tunnel-boot + // must be enabled so it runs on boot. The systemd unit is renamed; the + // WireGuard interface it brings up is still wg-relay. + bootUnit := string(fake.Files["/etc/systemd/system/tunnel-boot.service"]) + if !strings.Contains(bootUnit, "ExecStart="+tunnelctlBinPath+" apply --config "+relayDocumentPath) { + t.Fatalf("tunnel-boot.service missing tunnelctl apply ExecStart; got:\n%s", bootUnit) } - if !strings.Contains(wgUnit, "Type=oneshot") || !strings.Contains(wgUnit, "WantedBy=multi-user.target") { - t.Fatalf("wg-relay.service is not a boot oneshot; got:\n%s", wgUnit) + if !strings.Contains(bootUnit, "Type=oneshot") || !strings.Contains(bootUnit, "WantedBy=multi-user.target") { + t.Fatalf("tunnel-boot.service is not a boot oneshot; got:\n%s", bootUnit) } envoyUnit := string(fake.Files["/etc/systemd/system/envoy.service"]) - if !strings.Contains(envoyUnit, "Requires=wg-relay.service") { - t.Fatalf("envoy.service must require wg-relay.service; got:\n%s", envoyUnit) + if !strings.Contains(envoyUnit, "Requires=tunnel-boot.service") { + t.Fatalf("envoy.service must require tunnel-boot.service; got:\n%s", envoyUnit) + } + if !strings.Contains(joined, "systemctl enable tunnel-boot.service") { + t.Fatalf("tunnel-boot.service was not enabled; runs:\n%s", joined) + } + + // The sysctl drop-in is a single overwrite carrying ip_forward and the + // socket-buffer ceiling threaded from the plan, applied with sysctl -p. + sysctl := string(fake.Files[sysctlDropInPath]) + for _, want := range []string{ + "net.ipv4.ip_forward=1", + "net.core.rmem_max=8388608", + "net.core.wmem_max=8388608", + } { + if !strings.Contains(sysctl, want) { + t.Fatalf("sysctl drop-in missing %q; got:\n%s", want, sysctl) + } } - if !strings.Contains(joined, "systemctl enable wg-relay.service") { - t.Fatalf("wg-relay.service was not enabled; runs:\n%s", joined) + if !strings.Contains(joined, "sysctl -p "+sysctlDropInPath) { + t.Fatalf("sysctl drop-in was not applied with sysctl -p; runs:\n%s", joined) } // FIX m-3: the sweep must cover /etc/systemd/system where unit temp files land. diff --git a/internal/provision/teardown.go b/internal/provision/teardown.go index 994e64f..75c0c74 100644 --- a/internal/provision/teardown.go +++ b/internal/provision/teardown.go @@ -12,7 +12,7 @@ import ( // Teardown cleans up the VPS when the EdgeNode is deleted. It reverses what // Enroll provisions: it removes the WireGuard relay interface and Envoy -// service plus their configuration, the systemd unit, the sysctl drop-in, the +// service plus their configuration, the systemd units, the sysctl drop-in, the // tunnelctl binary and the operator state directory. // // Service-stop commands are made idempotent with a trailing "|| true" so that @@ -28,18 +28,19 @@ func Teardown(ctx context.Context, exec sshexec.Executor) error { return fmt.Errorf("failed to remove tunnelctl binary: %w", err) } - // Envoy service and configuration (envoy.yaml, lds.yaml, cds.yaml) and wg-relay service + // Envoy service and configuration (envoy.yaml, lds.yaml, cds.yaml) and the + // tunnel-boot oneshot. if _, err := exec.Run(ctx, "systemctl disable --now envoy || true"); err != nil { return fmt.Errorf("failed to stop envoy: %w", err) } - if _, err := exec.Run(ctx, "systemctl disable --now wg-relay.service || true"); err != nil { - return fmt.Errorf("failed to stop wg-relay.service: %w", err) + if _, err := exec.Run(ctx, "systemctl disable --now tunnel-boot.service || true"); err != nil { + return fmt.Errorf("failed to stop tunnel-boot.service: %w", err) } if _, err := exec.Run(ctx, "rm -f /etc/systemd/system/envoy.service"); err != nil { return fmt.Errorf("failed to remove envoy.service: %w", err) } - if _, err := exec.Run(ctx, "rm -f /etc/systemd/system/wg-relay.service"); err != nil { - return fmt.Errorf("failed to remove wg-relay.service: %w", err) + if _, err := exec.Run(ctx, "rm -f /etc/systemd/system/tunnel-boot.service"); err != nil { + return fmt.Errorf("failed to remove tunnel-boot.service: %w", err) } if _, err := exec.Run(ctx, "systemctl daemon-reload || true"); err != nil { return fmt.Errorf("failed to reload systemd: %w", err) diff --git a/internal/render/envoy.go b/internal/render/envoy.go index 29055c2..0eecb47 100644 --- a/internal/render/envoy.go +++ b/internal/render/envoy.go @@ -13,6 +13,7 @@ import ( const ldsTpl = `resources: {{- if .Listeners }} +{{- $buf := .UDPSocketBufferBytes }} {{- range .Listeners }} - "@type": type.googleapis.com/envoy.config.listener.v3.Listener name: listener_{{ .Name }} @@ -23,10 +24,25 @@ const ldsTpl = `resources: {{- if eq .Protocol "UDP" }} protocol: UDP {{- end }} + # enable_reuse_port lets the kernel load-balance across worker sockets; the + # older reuse_port field is deprecated and on Linux this is honored for both + # TCP and UDP. + enable_reuse_port: true {{- if eq .Protocol "UDP" }} - udp_listener_config: - downstream_socket_config: - prefer_gro: true + {{- if gt $buf 0 }} + # SO_RCVBUF (8) and SO_SNDBUF (7) at SOL_SOCKET (level 1), set before bind. + # UDP does not autotune its socket buffers, so they are raised explicitly to + # the configured kernel ceiling for high-throughput UDP listeners. + socket_options: + - level: 1 + name: 8 + int_value: {{ $buf }} + state: STATE_PREBIND + - level: 1 + name: 7 + int_value: {{ $buf }} + state: STATE_PREBIND + {{- end }} listener_filters: - name: envoy.filters.udp_listener.udp_proxy typed_config: diff --git a/internal/render/envoy_test.go b/internal/render/envoy_test.go index d772e37..45c9e04 100644 --- a/internal/render/envoy_test.go +++ b/internal/render/envoy_test.go @@ -144,6 +144,85 @@ func TestRenderEnvoyLDSAndCDS(t *testing.T) { } } +// TestRenderEnvoyLDS_UDPSocketBuffer verifies that a non-zero UDPSocketBufferBytes +// renders SO_RCVBUF/SO_SNDBUF socket_options on UDP listeners only, that TCP +// listeners never get socket_options (TCP autotunes), that every listener carries +// enable_reuse_port, and that prefer_gro is gone (delegated to the OS). A zero +// buffer renders no socket_options at all. +func TestRenderEnvoyLDS_UDPSocketBuffer(t *testing.T) { + cfg := EnvoyConfig{ + UDPSocketBufferBytes: 26214400, + Listeners: []EnvoyListener{ + { + Name: "udp_bind", + Protocol: "UDP", + ListenPort: 51820, + Upstreams: []EnvoyUpstreamServer{{"10.200.0.2", 51820}}, + UDP: EnvoyUDPParams{SessionTimeout: "60s"}, + }, + { + Name: "tcp_bind", + Protocol: "TCP", + ListenPort: 443, + Upstreams: []EnvoyUpstreamServer{{"10.200.0.2", 443}}, + TCP: EnvoyTCPParams{ConnectTimeout: "5s", IdleTimeout: "3600s"}, + }, + }, + } + + out, err := RenderEnvoyLDS(cfg) + if err != nil { + t.Fatal(err) + } + + out2, err := RenderEnvoyLDS(cfg) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(out, out2) { + t.Fatal("RenderEnvoyLDS with socket buffer is not deterministic") + } + + if bytes.Contains(out, []byte("prefer_gro")) { + t.Fatal("prefer_gro must be removed from the UDP listener (delegated to the OS)") + } + if n := bytes.Count(out, []byte("enable_reuse_port: true")); n != 2 { + t.Fatalf("expected enable_reuse_port on every listener (2), got %d", n) + } + if bytes.Count(out, []byte("socket_options")) != 1 { + t.Fatalf("expected exactly one socket_options block (UDP only), got:\n%s", out) + } + for _, want := range []string{"name: 8", "name: 7", "int_value: 26214400", "state: STATE_PREBIND"} { + if !bytes.Contains(out, []byte(want)) { + t.Fatalf("UDP socket_options missing %q;\n%s", want, out) + } + } + + // The TCP listener block must not carry socket_options. TCP (443) sorts + // before UDP (51820), so bound the TCP block at the next listener. + tcpIdx := bytes.Index(out, []byte("listener_tcp_bind")) + udpIdx := bytes.Index(out, []byte("listener_udp_bind")) + if tcpIdx == -1 || udpIdx == -1 { + t.Fatal("missing a listener in output") + } + if tcpIdx > udpIdx { + t.Fatalf("expected TCP listener before UDP listener in sorted output") + } + if bytes.Contains(out[tcpIdx:udpIdx], []byte("socket_options")) { + t.Fatal("TCP listener must not receive socket_options (TCP autotunes)") + } + + // A zero buffer renders no socket_options anywhere. + cfg.UDPSocketBufferBytes = 0 + zeroOut, err := RenderEnvoyLDS(cfg) + if err != nil { + t.Fatal(err) + } + if bytes.Contains(zeroOut, []byte("socket_options")) { + t.Fatal("no socket_options should render when the buffer is zero") + } +} + // TestRenderEnvoySDS verifies the SDS document embeds the cert/key inline for // offload, adds a CA validation resource for mutual, is deterministic, and // rejects incomplete input. diff --git a/internal/render/envoy_validate_test.go b/internal/render/envoy_validate_test.go index b6c9697..24cee44 100644 --- a/internal/render/envoy_validate_test.go +++ b/internal/render/envoy_validate_test.go @@ -230,6 +230,7 @@ func TestEnvoyValidate(t *testing.T) { } cfg := EnvoyConfig{ + UDPSocketBufferBytes: 26214400, Listeners: []EnvoyListener{ { Name: "tcp_plain", diff --git a/internal/render/render_types.go b/internal/render/render_types.go index 1cf829f..fde5298 100644 --- a/internal/render/render_types.go +++ b/internal/render/render_types.go @@ -117,4 +117,12 @@ type EnvoyListener struct { type EnvoyConfig struct { // Listeners is the complete list of port bindings to render. Listeners []EnvoyListener + + // UDPSocketBufferBytes is the SO_RCVBUF/SO_SNDBUF value applied to every UDP + // listener via socket_options. UDP sockets do not autotune, so this raises + // their buffers up to the kernel ceiling (the planner threads the configured + // host buffer here). When zero, no socket_options block is rendered and the + // OS defaults apply. TCP listeners never receive socket_options (TCP + // autotunes), so this value only affects UDP. + UDPSocketBufferBytes int64 } diff --git a/internal/render/testdata/envoy_lds.golden b/internal/render/testdata/envoy_lds.golden index 5551eba..dccd504 100644 --- a/internal/render/testdata/envoy_lds.golden +++ b/internal/render/testdata/envoy_lds.golden @@ -5,6 +5,10 @@ resources: socket_address: address: 0.0.0.0 port_value: 443 + # enable_reuse_port lets the kernel load-balance across worker sockets; the + # older reuse_port field is deprecated and on Linux this is honored for both + # TCP and UDP. + enable_reuse_port: true filter_chains: - filters: - name: envoy.filters.network.tcp_proxy @@ -20,9 +24,10 @@ resources: address: 0.0.0.0 port_value: 51820 protocol: UDP - udp_listener_config: - downstream_socket_config: - prefer_gro: true + # enable_reuse_port lets the kernel load-balance across worker sockets; the + # older reuse_port field is deprecated and on Linux this is honored for both + # TCP and UDP. + enable_reuse_port: true listener_filters: - name: envoy.filters.udp_listener.udp_proxy typed_config: diff --git a/internal/render/testdata/envoy_lds_tls_mutual.golden b/internal/render/testdata/envoy_lds_tls_mutual.golden index 926fd8c..88d462f 100644 --- a/internal/render/testdata/envoy_lds_tls_mutual.golden +++ b/internal/render/testdata/envoy_lds_tls_mutual.golden @@ -5,6 +5,10 @@ resources: socket_address: address: 0.0.0.0 port_value: 9443 + # enable_reuse_port lets the kernel load-balance across worker sockets; the + # older reuse_port field is deprecated and on Linux this is honored for both + # TCP and UDP. + enable_reuse_port: true filter_chains: - filters: - name: envoy.filters.network.tcp_proxy diff --git a/internal/render/testdata/envoy_lds_tls_offload.golden b/internal/render/testdata/envoy_lds_tls_offload.golden index 8404641..a7759a6 100644 --- a/internal/render/testdata/envoy_lds_tls_offload.golden +++ b/internal/render/testdata/envoy_lds_tls_offload.golden @@ -5,6 +5,10 @@ resources: socket_address: address: 0.0.0.0 port_value: 8443 + # enable_reuse_port lets the kernel load-balance across worker sockets; the + # older reuse_port field is deprecated and on Linux this is honored for both + # TCP and UDP. + enable_reuse_port: true filter_chains: - filters: - name: envoy.filters.network.tcp_proxy diff --git a/internal/render/testdata/envoy_lds_tls_passthrough.golden b/internal/render/testdata/envoy_lds_tls_passthrough.golden index c0481f5..94ac72a 100644 --- a/internal/render/testdata/envoy_lds_tls_passthrough.golden +++ b/internal/render/testdata/envoy_lds_tls_passthrough.golden @@ -5,6 +5,10 @@ resources: socket_address: address: 0.0.0.0 port_value: 443 + # enable_reuse_port lets the kernel load-balance across worker sockets; the + # older reuse_port field is deprecated and on Linux this is honored for both + # TCP and UDP. + enable_reuse_port: true listener_filters: - name: envoy.filters.listener.tls_inspector typed_config: