Add RouteAdvertisements for no-overlay CUDNs

Add per-network FRRConfiguration selector config so unmanaged
no-overlay UDN networks can create RouteAdvertisements before
waiting for transport acceptance.

Render RouteAdvertisements for primary and secondary CUDNs using
the UDN network labels, and clean them up with the other UDN
cluster-scoped resources.

Signed-off-by: Venkat Kunaparaju <vkunaparaju@nvidia.com>

Author: vkunaparaju

README.md

@@ -83,9 +83,11 @@ tft:
       mode: "(33)"
       topology: "(34)"
       transport: "(35)"
-kubeconfig: (36)
-kubeconfig_infra: (36)
-dpu_node_host_label: (37)
+      frr_configuration_selector: # (36)
+        "(37)": "(38)"
+kubeconfig: (39)
+kubeconfig_infra: (39)
+dpu_node_host_label: (40)
 ```
 
 1. "name" - This is the name of the test. Any string value to identify the test.
@@ -206,10 +208,13 @@ dpu_node_host_label: (37)
 33. "mode" - (Optional) Field under `udn_primary_network`. Supported values are `udn` and `cudn`.
 34. "topology" - (Optional) Field under `udn_primary_network`. Supported values are `layer3` and `layer2`.
 35. "transport" - (Optional) Field under `udn_primary_network`. Supported values are `overlay` and `no-overlay`; `no-overlay` requires `mode: cudn` and `topology: layer3`.
-36. "kubeconfig", "kubeconfig_infra": if set to non-empty strings, then these are the KUBECONFIG
+36. "frr_configuration_selector" - (Optional) Field under `udn_primary_network`. Map of `frrConfigurationSelector.matchLabels` labels used to create RouteAdvertisements for unmanaged no-overlay CUDNs. If omitted or empty, RouteAdvertisements are not created.
+37. selector label key - A Kubernetes label key under `frr_configuration_selector`.
+38. selector label value - A Kubernetes label value under `frr_configuration_selector`. Empty string values are supported.
+39. "kubeconfig", "kubeconfig_infra": if set to non-empty strings, then these are the KUBECONFIG
   files. "kubeconfig_infra" must be set for DPU cluster mode. If both are empty, the configs
   are detected based on the files we find at /root/kubeconfig.*.
-37. "dpu_node_host_label": (Required for DPU mode) The label on DPU nodes that identifies
+40. "dpu_node_host_label": (Required for DPU mode) The label on DPU nodes that identifies
   which host worker node they belong to. For NVIDIA DPUs, use `provisioning.dpu.nvidia.com/host`.
 
 
@@ -235,6 +240,18 @@ The primary CIDR defaults to `15.1.0.0/16`. Secondary CIDRs default to `15.2.0.0
 
 `udn_primary_network` supports `mode` values `udn` and `cudn`, `topology` values `layer3` and `layer2`, and `transport` values `overlay` and `no-overlay`. `no-overlay` requires `mode: cudn` and `topology: layer3`.
 
+For unmanaged no-overlay primary CUDNs, set `frr_configuration_selector` to create a RouteAdvertisements object that advertises the selected CUDN pod network. The selector is a map of `frrConfigurationSelector.matchLabels` labels. When `TFT_UDN_NO_OVERLAY_ROUTING_MANAGED` is true, RouteAdvertisements are not created.
+
+```yaml
+udn_primary_network:
+  mode: cudn
+  topology: layer3
+  transport: no-overlay
+  frr_configuration_selector:
+    network: blue
+    ra.k8s.ovn.org/example: ""
+```
+
 ## DPU Mode
 
 When running with a DPU (Data Processing Unit) cluster, the `validate_offload` plugin needs

manifests/udn-route-advertisements.yaml.j2

@@ -0,0 +1,22 @@
+apiVersion: k8s.ovn.org/v1
+kind: RouteAdvertisements
+metadata:
+  name: {{ network_name }}
+  labels:
+    tft-tests: "udn"
+spec:
+  advertisements:
+  - PodNetwork
+  frrConfigurationSelector:
+    matchLabels:
+    {%- for key, value in frr_configuration_selector %}
+      {{ key }}: {{ value }}
+    {%- endfor %}
+  networkSelectors:
+  - clusterUserDefinedNetworkSelector:
+      networkSelector:
+        matchLabels:
+          tft-tests: "udn"
+          tft-udn-network: {{ network_label }}
+    networkSelectionType: ClusterUserDefinedNetworks
+  nodeSelector: {}

testConfig.py

@@ -98,6 +98,24 @@ def _construct_capabilities_pod(
     return {"add": tuple(arg["add"])}
 
 
+def _construct_str_mapping(pctx: StructParseParseContext) -> Mapping[str, str]:
+    arg = common.structparse_check_strdict(pctx.arg, pctx.yamlpath)
+    result: dict[str, str] = {}
+    for key, value in arg.items():
+        if not key:
+            raise pctx.value_error("label key cannot be empty")
+        if value is None:
+            result[key] = ""
+        elif isinstance(value, str):
+            result[key] = value
+        else:
+            raise pctx.value_error(
+                f"expects string label values but got {repr(value)}",
+                key=key,
+            )
+    return result
+
+
 T2 = TypeVar("T2", bound="ConfNodeServer | ConfNodeClient")
 
 
@@ -602,6 +620,7 @@ class ConfUdnNetwork(StructParseBase):
     mode: UdnNetworkMode
     topology: UdnNetworkTopology
     transport: Optional[UdnNetworkTransport]
+    frr_configuration_selector: Mapping[str, str]
 
     def serialize(self) -> dict[str, Any]:
         data: dict[str, Any] = {
@@ -610,6 +629,8 @@ def serialize(self) -> dict[str, Any]:
         }
         if self.transport is not None:
             data["transport"] = self.transport.name
+        if self.frr_configuration_selector:
+            data["frr_configuration_selector"] = dict(self.frr_configuration_selector)
         return data
 
     @staticmethod
@@ -626,6 +647,7 @@ def parse(
             mode = UdnNetworkMode.UDN
             topology = default_topology
             transport: Optional[UdnNetworkTransport] = UdnNetworkTransport.OVERLAY
+            frr_configuration_selector: Mapping[str, str] = {}
         else:
             with pctx.with_strdict() as varg:
                 transport_configured = "transport" in varg.vdict
@@ -644,6 +666,11 @@ def parse(
                     enum_type=UdnNetworkTransport,
                     default=None,
                 )
+                frr_configuration_selector = common.structparse_pop_obj(
+                    varg.for_key("frr_configuration_selector"),
+                    construct=_construct_str_mapping,
+                    default={},
+                )
             if transport is None and topology != UdnNetworkTopology.LOCALNET:
                 transport = UdnNetworkTransport.OVERLAY
             if topology == UdnNetworkTopology.LOCALNET and transport_configured:
@@ -674,13 +701,19 @@ def parse(
                 "no-overlay transport is only supported for layer3 primary CUDN",
                 key="transport",
             )
+        if frr_configuration_selector and transport != UdnNetworkTransport.NO_OVERLAY:
+            raise pctx.value_error(
+                "frr_configuration_selector requires transport no-overlay",
+                key="frr_configuration_selector",
+            )
 
         return ConfUdnNetwork(
             yamlidx=pctx.yamlidx,
             yamlpath=pctx.yamlpath,
             mode=mode,
             topology=topology,
             transport=transport,
+            frr_configuration_selector=frr_configuration_selector,
         )
 
 

tests/test_testConfig.py

@@ -430,6 +430,7 @@ def test_udn_primary_network_defaults_preserve_existing_behavior() -> None:
     assert primary.mode == UdnNetworkMode.UDN
     assert primary.topology == UdnNetworkTopology.LAYER3
     assert primary.transport == UdnNetworkTransport.OVERLAY
+    assert primary.frr_configuration_selector == {}
 
     _check_testConfig(tc)
 
@@ -443,6 +444,9 @@ def test_udn_primary_network_config() -> None:
       mode: cudn
       topology: layer3
       transport: no-overlay
+      frr_configuration_selector:
+        network: blue
+        ra.k8s.ovn.org/example: ""
     connections:
     - {}
 """)
@@ -455,6 +459,10 @@ def test_udn_primary_network_config() -> None:
     assert primary.mode == UdnNetworkMode.CUDN
     assert primary.topology == UdnNetworkTopology.LAYER3
     assert primary.transport == UdnNetworkTransport.NO_OVERLAY
+    assert primary.frr_configuration_selector == {
+        "network": "blue",
+        "ra.k8s.ovn.org/example": "",
+    }
 
     _check_testConfig(tc)
 

trafficFlowTests.py

@@ -85,6 +85,45 @@ def _wait_for_cudn_transport_accepted(
             f"ClusterUserDefinedNetwork/{cudn_name} TransportAccepted not True, continuing"
         )
 
+    def _setup_udn_route_advertisements(
+        self,
+        cfg_descr: ConfigDescriptor,
+        *,
+        network: testConfig.ConfUdnNetwork,
+        network_name: str,
+        network_label: str,
+    ) -> None:
+        if not network.frr_configuration_selector:
+            logger.info(
+                "No no-overlay FRRConfiguration selector configured; "
+                f"not creating RouteAdvertisements for {network_name}"
+            )
+            return
+
+        client = cfg_descr.tc.client_tenant
+        in_template = tftbase.get_manifest("udn-route-advertisements.yaml.j2")
+        out_yaml = tftbase.get_manifest_renderpath(
+            f"udn-route-advertisements-{network_label}.yaml"
+        )
+        _j = json.dumps
+        kjinja2.render_file(
+            in_template,
+            {
+                "network_name": _j(network_name),
+                "network_label": _j(network_label),
+                "frr_configuration_selector": [
+                    (_j(k), _j(v))
+                    for k, v in network.frr_configuration_selector.items()
+                ],
+            },
+            out_file=out_yaml,
+        )
+        logger.info(
+            f'Generate {network_label} RouteAdvertisements "{out_yaml}" '
+            f'(from "{in_template}")'
+        )
+        client.oc(f"apply -f {out_yaml}", die_on_error=True)
+
     def _setup_udn_network(
         self,
         cfg_descr: ConfigDescriptor,
@@ -147,6 +186,13 @@ def _setup_udn_network(
         )
         logger.info(f'Generate {network_label} "{out_yaml}" (from "{in_template}")')
         client.oc(f"apply -f {out_yaml}", die_on_error=True)
+        if is_no_overlay and not routing_managed:
+            self._setup_udn_route_advertisements(
+                cfg_descr,
+                network=network,
+                network_name=network_name,
+                network_label=network_label,
+            )
         if is_no_overlay:
             self._wait_for_cudn_transport_accepted(cfg_descr, network_name)
 
@@ -218,6 +264,7 @@ def _setup_udn(self, cfg_descr: ConfigDescriptor) -> None:
                 mode=network.mode,
                 topology=network.topology,
                 transport=network.transport,
+                frr_configuration_selector={},
             )
             self._setup_udn_network(
                 cfg_descr,
@@ -358,6 +405,11 @@ def _cleanup_stale_udn(self, cfg_descr: ConfigDescriptor) -> None:
             namespace=None,
             may_fail=True,
         )
+        client.oc(
+            "delete routeadvertisements -l tft-tests",
+            namespace=None,
+            may_fail=True,
+        )
         if client.oc_get(f"namespace/{udn_ns}", may_fail=True) is None:
             return
         logger.info(
@@ -457,6 +509,11 @@ def _cleanup_udn(self, cfg_descr: ConfigDescriptor) -> None:
             namespace=None,
             may_fail=True,
         )
+        client.oc(
+            "delete routeadvertisements -l tft-tests",
+            namespace=None,
+            may_fail=True,
+        )
         client.oc(f"delete userdefinednetwork -l tft-tests -n {udn_ns}", may_fail=True)
         if self._udn_ns_created:
             logger.info(f"Deleting UDN namespace {udn_ns}")