confd: allow both ends of a veth pair to be assigned to containers

Previously at least one end of a veth pair had to remain in the host
namespace, because that end created and destroyed the pair.  Assigning
both ends to containers left no one to create it.

Select a deterministic primary end so exactly one side creates the pair.
When the primary is itself a container interface, create the pair in the
host namespace before the container starts; CNI host-device then moves
each end into its container.  Teardown is deferred to the container
removal script so the pair does not linger and block re-creation.

Drop the now-obsolete limitation notes from the documentation and YANG,
and add a regression test connecting two containers over a veth pair.

Fixes: #941

Signed-off-by: Joachim Wiberg <troglobit@gmail.com>

Author: troglobit

doc/ChangeLog.md

@@ -30,6 +30,8 @@ All notable changes to the project are documented in this file.
 
 ### Fixes
 
+- Fix #941: a VETH pair can now connect two containers directly, with both
+  ends assigned to containers.
 - Enabling IP masquerading in the firewall no longer enables IP forwarding on
   all interfaces.  This has been an issue ever since the firewall support was
   introduced in v25.10.0

doc/container.md

@@ -668,11 +668,9 @@ set:
 
 For an example of both, see the next section.
 
-> [!IMPORTANT]
-> **VETH Pair Limitation:** When using VETH pairs with containers, at least
-> one side of the pair must remain in the host namespace. It is currently
-> not possible to create VETH pairs where both ends are assigned to different
-> containers. One end must always be accessible from the host.
+> [!TIP]
+> Both ends of a VETH pair may be assigned to containers, connecting two
+> containers directly without involving the host namespace.
 
 [^3]: Something which the container bridge network type does behind the
     scenes with one end of an automatically created VETH pair.

src/confd/src/cni.c

@@ -414,6 +414,16 @@ int cni_netdag_gen_iface(struct dagger *net, const char *ifname,
 			return -EIO;
 
 		fprintf(fp, "container -a -f delete network %s >/dev/null\n", ifname);
+
+		/* If this end belongs to a veth pair, the kernel keeps the pair
+		 * alive after CNI host-device returns the interface to the host
+		 * namespace.  Remove it here, once the container is gone, so the
+		 * pair does not linger and block a later re-creation.  Tolerant:
+		 * the peer's teardown may already have removed it.
+		 */
+		if (lydx_get_child(dif, "veth"))
+			fprintf(fp, "ip link del dev %s 2>/dev/null || true\n", ifname);
+
 		fclose(fp);
 
 		if (cni_type == IFT_BRIDGE)

src/confd/src/if-veth.c

@@ -21,23 +21,28 @@
 bool veth_is_primary(struct lyd_node *cif)
 {
 	struct lyd_node *peer, *veth;
+	bool self_cni, peer_cni;
 	const char *peername;
 
 	veth = lydx_get_child(cif, "veth");
 	peername = lydx_get_cattr(veth, "peer");
 	peer = lydx_find_by_name(lyd_parent(cif), "interface", peername);
 
-	/* At the moment, CNI code relies on one side of the pair
-	 * remaining in the host namespace, and that that interface
-	 * takes care of creating the pair.
+	self_cni = lydx_get_child(cif, "container-network") != NULL;
+	peer_cni = lydx_get_child(peer, "container-network") != NULL;
+
+	/* When exactly one end is handed to a container (CNI host-device),
+	 * the other end stays in the host namespace and creates the pair.
 	 */
-	if (lydx_get_child(cif, "container-network"))
-		return false;
-	if (lydx_get_child(peer, "container-network"))
-		return true;
+	if (self_cni != peer_cni)
+		return peer_cni;
 
-	return strcmp(lydx_get_cattr(cif, "name"),
-		      lydx_get_cattr(veth, "peer")) < 0;
+	/* Neither or both ends are container interfaces: pick a stable
+	 * primary by name so exactly one end creates the pair.  When both
+	 * ends are containers the pair is still created in the host
+	 * namespace first, then moved into each container by CNI host-device.
+	 */
+	return strcmp(lydx_get_cattr(cif, "name"), peername) < 0;
 }
 
 int ifchange_cand_infer_veth(sr_session_ctx_t *session, const char *path)

src/confd/src/interfaces.c

@@ -518,6 +518,14 @@ static int veth_gen_del(struct lyd_node *dif, FILE *sh)
 	if (!veth_is_primary(dif))
 		return 0;
 
+	/* When the primary end is itself a container interface it currently
+	 * lives in the container's namespace, so a host-namespace delete here
+	 * would fail and abort the teardown.  Its removal is handled after the
+	 * container is gone, see cni_netdag_gen_iface().
+	 */
+	if (lydx_get_child(dif, "container-network"))
+		return 0;
+
 	return link_gen_del(dif, sh);
 }
 
@@ -571,6 +579,28 @@ static int netdag_gen_iface_del(struct dagger *net, struct lyd_node *dif,
 	return 0;
 }
 
+/*
+ * Both ends of a veth pair can be handed to containers, leaving no
+ * host-side interface to create the pair.  Have the primary end create it
+ * in the host namespace early (NETDAG_INIT_PHYS, before the container is
+ * (re)started); CNI host-device then moves each end into its container.
+ */
+static int veth_gen_host(struct dagger *net, struct lyd_node *dif, struct lyd_node *cif)
+{
+	const char *ifname = lydx_get_cattr(cif, "name");
+	FILE *ip;
+	int err;
+
+	ip = dagger_fopen_net_init(net, ifname, NETDAG_INIT_PHYS, "init.ip");
+	if (!ip)
+		return -EIO;
+
+	err = veth_gen(dif, cif, ip);
+	fclose(ip);
+
+	return err;
+}
+
 static sr_error_t netdag_gen_iface_timeout(struct dagger *net, const char *ifname, const char *iftype)
 {
 	if (!strcmp(iftype, "infix-if-type:ethernet")) {
@@ -604,8 +634,13 @@ static sr_error_t netdag_gen_iface(sr_session_ctx_t *session, struct dagger *net
 
 	if ((err = cni_netdag_gen_iface(net, ifname, dif, cif))) {
 		/* error or managed by CNI/podman */
-		if (err > 0)
+		if (err > 0) {
 			err = 0; /* done, nothing more to do here */
+
+			if (op == LYDX_OP_CREATE && lydx_get_child(cif, "veth") &&
+			    veth_is_primary(cif))
+				err = veth_gen_host(net, dif, cif);
+		}
 		goto err;
 	}
 

src/confd/yang/confd/infix-if-container.yang

@@ -59,11 +59,7 @@ submodule infix-if-container {
 
   identity host {
     base container-network;
-    description "Host device, e.g., one end of a VETH pair or other host interface.
-
-                 Note: When using VETH pairs, at least one side must remain in the
-                 host namespace. Both ends of a VETH pair cannot be assigned to
-                 different containers.";
+    description "Host device, e.g., one end of a VETH pair or other host interface.";
   }
 
   /*

src/confd/yang/confd/infix-if-veth.yang

@@ -13,11 +13,7 @@ submodule infix-if-veth {
 
   organization "KernelKit";
   contact      "kernelkit@googlegroups.com";
-  description  "Linux virtual Ethernet pair extension for ietf-interfaces.
-
-                Note: When using VETH pairs with containers, at least one side
-                of the pair must remain in the host namespace. Both ends of a
-                VETH pair cannot be assigned to different containers.";
+  description  "Linux virtual Ethernet pair extension for ietf-interfaces.";
 
   revision 2023-06-05 {
     description "Initial revision.";

test/case/containers/all.yaml

@@ -18,6 +18,9 @@
 - name: Container with VETH Pair
   case: veth/test.py
 
+- name: VETH Pair Between Two Containers
+  case: internal_link/test.py
+
 - name: Container Volume Persistence
   case: volume/test.py
 

test/case/containers/internal_link/Readme.adoc

@@ -0,0 +1 @@
+test.adoc

test/case/containers/internal_link/test.adoc

@@ -0,0 +1,33 @@
+=== VETH Pair Between Two Containers
+
+ifdef::topdoc[:imagesdir: {topdoc}../../test/case/containers/internal_link]
+
+==== Description
+
+Verify that a VETH pair can connect two containers directly, with *both*
+ends handed to containers and neither remaining in the host namespace.
+
+....
+  .------------.                          .------------.
+  |    left    |                          |    right   |
+  |  veth0a ===|========= veth ===========|=== veth0b  |
+  '------------' 10.0.0.1        10.0.0.2 '------------'
+....
+
+The pair is created in the host namespace then each end is moved into
+its container when starting up.  Connectivity is verified by pinging
+across the pair, from inside one container's network namespace to the
+other end's address.
+
+==== Topology
+
+image::topology.svg[VETH Pair Between Two Containers topology, align=center, scaledwidth=75%]
+
+==== Sequence
+
+. Set up topology and attach to target DUT
+. Create VETH pair with both ends assigned to containers
+. Verify both containers have started
+. Verify {LEFT} reaches {RIGHT} over the internal VETH pair
+
+