Replace rootlessport with pesto for rootless bridge port forwarding

Pesto dynamically updates pasta's forwarding table via UNIX socket,
preserving source IPs that rootlessport's userspace proxy destroyed.

Strip HostIP from port mappings passed to netavark in rootless mode
(portMappingsForNetavark) because pesto handles host-side address
binding via pasta. Netavark's DNAT rules inside the rootless netns
must not restrict on destination address since pasta's splice delivers
traffic with a different address than the user-specified HostIP.
Pesto receives the original HostIP so pasta binds correctly on the
host.

Fixes: https://redhat.atlassian.net/browse/RUN-2214
Fixes: #8193
Fixes: https://redhat.atlassian.net/browse/RUN-3587

Signed-off-by: Jan Rodák <hony.com@seznam.cz>

Author: Honny1

libpod/networking_common.go

@@ -59,7 +59,7 @@ func (c *Container) getNetworkOptions(networkOpts map[string]types.PerNetworkOpt
 		DNSServers:        nameservers,
 		ContainerHostname: c.NetworkHostname(),
 	}
-	opts.PortMappings = c.convertPortMappings()
+	opts.PortMappings = portMappingsForNetavark(c.convertPortMappings())
 
 	// If the container requested special network options use this instead of the config.
 	// This is the case for container restore or network reload.
@@ -71,6 +71,25 @@ func (c *Container) getNetworkOptions(networkOpts map[string]types.PerNetworkOpt
 	return opts
 }
 
+// portMappingsForNetavark strips HostIP from port mappings in rootless mode.
+//
+// Pesto handles host-side address binding; netavark creates DNAT rules inside
+// the rootless netns. If HostIP is kept, netavark adds an "ip daddr <HostIP>"
+// constraint to the DNAT rule, but pasta's splice delivers traffic with a
+// different destination address (the host virtual IP), so the rule never
+// matches and the connection resets. Rootful is unaffected (no-op).
+func portMappingsForNetavark(ports []types.PortMapping) []types.PortMapping {
+	if !rootless.IsRootless() || len(ports) == 0 {
+		return ports
+	}
+	stripped := make([]types.PortMapping, len(ports))
+	copy(stripped, ports)
+	for i := range stripped {
+		stripped[i].HostIP = ""
+	}
+	return stripped
+}
+
 // setUpNetwork will set up the networks, on error it will also tear down the
 // networks. If rootless it will join/create the rootless network namespace.
 func (r *Runtime) setUpNetwork(ns string, opts types.NetworkOptions) (map[string]types.StatusBlock, error) {
@@ -110,11 +129,25 @@ func (r *Runtime) teardownNetwork(ctr *Container) error {
 		return err
 	}
 
-	if !ctr.config.NetMode.IsPasta() && len(networks) > 0 {
-		netOpts := ctr.getNetworkOptions(networks)
-		return r.teardownNetworkBackend(ctr.state.NetNS, netOpts)
+	if len(networks) == 0 {
+		return nil
 	}
-	return nil
+
+	// --net=pasta: per-container pasta cleans up when it exits, nothing to tear down.
+	if ctr.config.NetMode.IsPasta() {
+		return nil
+	}
+
+	// Bridge mode: update pesto before netavark so pasta stops forwarding
+	// ports for this container before the bridge/nftables rules are removed.
+	if rootless.IsRootless() && ctr.config.NetMode.IsBridge() && len(ctr.config.PortMappings) > 0 {
+		if err := r.teardownRootlessPortMappingViaPesto(ctr); err != nil {
+			logrus.Warnf("pesto port cleanup failed for container %s: %v", ctr.ID(), err)
+		}
+	}
+
+	netOpts := ctr.getNetworkOptions(networks)
+	return r.teardownNetworkBackend(ctr.state.NetNS, netOpts)
 }
 
 // isBridgeNetMode checks if the given network mode is bridge.
@@ -402,7 +435,7 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, _ bool) error {
 		ContainerID:   c.config.ID,
 		ContainerName: getNetworkPodName(c),
 	}
-	opts.PortMappings = c.convertPortMappings()
+	opts.PortMappings = portMappingsForNetavark(c.convertPortMappings())
 	opts.Networks = map[string]types.PerNetworkOptions{
 		netName: networks[netName],
 	}
@@ -421,11 +454,10 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, _ bool) error {
 		return err
 	}
 
-	// Reload ports when there are still connected networks, maybe we removed the network interface with the child ip.
-	// Reloading without connected networks does not make sense, so we can skip this step.
-	if rootless.IsRootless() && len(networkStatus) > 0 {
-		if err := c.reloadRootlessRLKPortMapping(); err != nil {
-			return err
+	// Update pesto's forwarding table after disconnect so pasta reflects the new network state.
+	if rootless.IsRootless() && len(networkStatus) > 0 && len(c.config.PortMappings) > 0 {
+		if err := c.runtime.setupRootlessPortMappingViaPesto(c); err != nil {
+			logrus.Warnf("pesto port reload failed after network disconnect for container %s: %v", c.ID(), err)
 		}
 	}
 
@@ -542,7 +574,7 @@ func (c *Container) NetworkConnect(nameOrID, netName string, netOpts types.PerNe
 		ContainerID:   c.config.ID,
 		ContainerName: getNetworkPodName(c),
 	}
-	opts.PortMappings = c.convertPortMappings()
+	opts.PortMappings = portMappingsForNetavark(c.convertPortMappings())
 	opts.Networks = map[string]types.PerNetworkOptions{
 		netName: netOpts,
 	}
@@ -575,11 +607,10 @@ func (c *Container) NetworkConnect(nameOrID, netName string, netOpts types.PerNe
 		return err
 	}
 
-	// The first network needs a port reload to set the correct child ip for the rootlessport process.
-	// Adding a second network does not require a port reload because the child ip is still valid.
-	if rootless.IsRootless() && len(networks) == 0 {
-		if err := c.reloadRootlessRLKPortMapping(); err != nil {
-			return err
+	// Update pesto's forwarding table after connect so pasta reflects the new network state.
+	if rootless.IsRootless() && len(c.config.PortMappings) > 0 {
+		if err := c.runtime.setupRootlessPortMappingViaPesto(c); err != nil {
+			logrus.Warnf("pesto port reload failed after network connect for container %s: %v", c.ID(), err)
 		}
 	}
 

libpod/networking_freebsd.go

@@ -224,3 +224,11 @@ func (c *Container) inspectJoinedNetworkNS(_ string) (q types.StatusBlock, retEr
 func (c *Container) reloadRootlessRLKPortMapping() error {
 	return errors.New("unsupported (*Container).reloadRootlessRLKPortMapping")
 }
+
+func (r *Runtime) setupRootlessPortMappingViaPesto(_ *Container) error {
+	return errors.New("unsupported setupRootlessPortMappingViaPesto on FreeBSD")
+}
+
+func (r *Runtime) teardownRootlessPortMappingViaPesto(_ *Container) error {
+	return errors.New("unsupported teardownRootlessPortMappingViaPesto on FreeBSD")
+}

libpod/networking_linux.go

@@ -59,15 +59,11 @@ func (r *Runtime) configureNetNS(ctr *Container, ctrNS string) (status map[strin
 		}
 	}()
 
-	// set up rootless port forwarder when rootless with ports and the network status is empty,
-	// if this is called from network reload the network status will not be empty and we should
-	// not set up port because they are still active
-	if rootless.IsRootless() && len(ctr.config.PortMappings) > 0 && ctr.getNetworkStatus() == nil {
-		// set up port forwarder for rootless netns
-		// make sure to fix this in container.handleRestartPolicy() as well
-		// Important we have to call this after r.setUpNetwork() so that
-		// we can use the proper netStatus
-		err = r.setupRootlessPortMappingViaRLK(ctr, ctrNS, netStatus)
+	// Set up port forwarding for rootless bridge networks via pesto.
+	// Pesto replaces the forwarding table on the running pasta instance,
+	// preserving source IPs.
+	if rootless.IsRootless() && len(ctr.config.PortMappings) > 0 {
+		err = r.setupRootlessPortMappingViaPesto(ctr)
 	}
 	return netStatus, err
 }

libpod/networking_pesto_linux.go

@@ -0,0 +1,97 @@
+//go:build !remote
+
+// Pesto integration for rootless bridge network port forwarding.
+//
+// A shared pasta instance in the rootless netns (-c pasta.sock) handles
+// host-side port forwarding. On container start/stop, pesto replaces
+// pasta's forwarding table with the aggregate ports of all running bridge
+// containers. Pasta forwards via kernel splice (localhost) or TAP
+// (external), preserving source IPs. The container sees the real client's
+// address instead of a proxy or bridge gateway address.
+//
+// Container start:
+// - netavark sets up bridge + DNAT
+// - pesto updates table
+//
+// Container stop:
+// - pesto updates table without stopped container's ports
+// - netavark tears down bridge/DNAT
+//
+// Limitations:
+//   - IPv4 only (netavark DNAT is IPv4; pesto binds 0.0.0.0 by default)
+//   - Full table replacement per change (brief gap, possible races)
+//   - gatherAllRootlessBridgePorts reads all containers from DB (no locks)
+
+package libpod
+
+import (
+	"github.com/containers/podman/v6/libpod/define"
+	"go.podman.io/common/libnetwork/pasta"
+	"go.podman.io/common/libnetwork/types"
+)
+
+// TODO: When pesto gains --add, --clear, --delete flags, switch from full
+// table replacement to incremental updates to avoid brief port interruptions
+// and reduce overhead with many container and possible race conditions.
+
+func (r *Runtime) pestoSocketPath() string {
+	info, err := r.network.RootlessNetnsInfo()
+	if err != nil || info == nil {
+		return ""
+	}
+	return info.PestoSocketPath
+}
+
+// setupRootlessPortMappingViaPesto configures port forwarding for a rootless
+// bridge container by updating the shared pasta instance's forwarding table.
+func (r *Runtime) setupRootlessPortMappingViaPesto(ctr *Container) error {
+	allPorts, err := r.gatherAllRootlessBridgePorts(ctr, true)
+	if err != nil {
+		return err
+	}
+	if len(allPorts) == 0 {
+		return nil
+	}
+
+	if err := pasta.PestoSetupPorts(r.config, r.pestoSocketPath(), allPorts); err != nil {
+		return err
+	}
+	return nil
+}
+
+// teardownRootlessPortMappingViaPesto removes a container's ports from pasta's forwarding table.
+func (r *Runtime) teardownRootlessPortMappingViaPesto(ctr *Container) error {
+	remainingPorts, err := r.gatherAllRootlessBridgePorts(ctr, false)
+	if err != nil {
+		return err
+	}
+	return pasta.PestoTeardownPorts(r.config, r.pestoSocketPath(), remainingPorts)
+}
+
+// gatherAllRootlessBridgePorts collects port mappings from all running
+// rootless bridge containers. When includeCtr is true, ctr's own ports
+// are included; when false they are excluded.
+func (r *Runtime) gatherAllRootlessBridgePorts(ctr *Container, includeCtr bool) ([]types.PortMapping, error) {
+	var allPorts []types.PortMapping
+
+	ctrs, err := r.state.AllContainers(true)
+	if err != nil {
+		return nil, err
+	}
+	for _, c := range ctrs {
+		if c.ID() == ctr.ID() {
+			continue
+		}
+		if c.state.State != define.ContainerStateRunning {
+			continue
+		}
+		if !c.config.NetMode.IsBridge() {
+			continue
+		}
+		allPorts = append(allPorts, c.convertPortMappings()...)
+	}
+	if includeCtr {
+		allPorts = append(allPorts, ctr.convertPortMappings()...)
+	}
+	return allPorts, nil
+}

test/e2e/common_test.go

@@ -1629,6 +1629,52 @@ func testPortConnection(port int) {
 	Expect(err).ToNot(HaveOccurred())
 }
 
+// sendMessageToPort sends a message to the given tcp port
+func sendMessageToPort(port int, message string) {
+	GinkgoHelper()
+	conn, err := net.DialTimeout("tcp", net.JoinHostPort("localhost", strconv.Itoa(port)), 5*time.Second)
+	Expect(err).ToNot(HaveOccurred(), "should connect to published port")
+
+	tcpConn := conn.(*net.TCPConn)
+	_, err = tcpConn.Write([]byte(message))
+	Expect(err).ToNot(HaveOccurred())
+
+	err = tcpConn.CloseWrite()
+	Expect(err).ToNot(HaveOccurred())
+
+	err = tcpConn.SetReadDeadline(time.Now().Add(5 * time.Second))
+	Expect(err).ToNot(HaveOccurred())
+	_, _ = io.Copy(io.Discard, tcpConn)
+	tcpConn.Close()
+}
+
+// startNCContainer starts a detached container running nc (netcat) listening
+// on the given port, waits for it to be ready, and returns the container name.
+func (p *PodmanTestIntegration) startNCContainer(name string, listenPort int, extraArgs ...string) string {
+	GinkgoHelper()
+	portStr := strconv.Itoa(listenPort)
+	args := append([]string{"run", "-d", "--name", name}, extraArgs...)
+	args = append(args, ALPINE, "sh", "-c", "nc -l -n -v -p "+portStr+" 2>&1")
+	p.PodmanExitCleanly(args...)
+	p.WaitForContainerLog(name, "listening")
+	return name
+}
+
+// WaitForContainerLog polls container logs until the given substring appears
+// in either stdout or stderr. Fails the test if not found within the timeout.
+func (p *PodmanTestIntegration) WaitForContainerLog(ctrName string, substr string) {
+	GinkgoHelper()
+	for range 10 {
+		logs := p.Podman([]string{"logs", ctrName})
+		logs.WaitWithDefaultTimeout()
+		if strings.Contains(logs.ErrorToString(), substr) || strings.Contains(logs.OutputToString(), substr) {
+			return
+		}
+		time.Sleep(500 * time.Millisecond)
+	}
+	Fail(fmt.Sprintf("timed out waiting for %q in logs of container %s", substr, ctrName))
+}
+
 func createNetworkName(name string) string {
 	return name + stringid.GenerateRandomID()[:10]
 }