Merge pull request #242 from aserdean/fixup-multi-networks

Make multi-network tests self-contained by auto-creating the secondary NAD

Author: wizhaoredhat

README.md

@@ -165,7 +165,7 @@ kubeconfig_infra: (29)
     | measure_power    | Measure Power Usage  |
     | validate_offload | Verify OvS Offload   |
 20a. "test_cases" - (Optional) Restrict a plugin to run only for the specified test cases. Uses the same format as the top-level `test_cases` field. By default, the plugin runs for every test case.
-21. "secondary_network_nad" - (Optional) - The name of the secondary network for multi-homing and multi-networkpolicies tests. For tests except 27-31, the primary network will be used if unspecified (the default which is None). For mandatory tests 27-31 it defaults to "tft-secondary" if not set. Can be overridden per-node using the server/client level `secondary_network_nad` fields.
+21. "secondary_network_nad" - (Optional) - The name of the secondary network for multi-homing and multi-networkpolicies tests. For tests except 27-31, the primary network will be used if unspecified (the default which is None). For mandatory tests 27-31 it defaults to "tft-secondary" if not set. Can be overridden per-node using the server/client level `secondary_network_nad` fields. The framework automatically creates and cleans up the NAD when these test cases are selected or when SRIOV nodes reference a secondary NAD. Subnets, MTU, and topology default to `10.193.0.0/16/26`, `1500`, and `layer3`, overridable via `TFT_SECONDARY_NAD_SUBNETS`, `TFT_SECONDARY_NAD_MTU`, and `TFT_SECONDARY_NAD_TOPOLOGY`.
 22. "resource_name" - (Optional) - The resource name for tests that require resource limit and requests to be set. This field is optional and will default to None if not set, but if secondary network nad is defined, traffic flow test
 tool will try to autopopulate resource_name based on the secondary+network_nad provided.
 23. "cpu_request" - (Optional) CPU request for server and client pods (e.g. "10m", "500m"). No CPU request is set if omitted.

manifests/secondary-nad.yaml.j2

@@ -0,0 +1,24 @@
+apiVersion: "k8s.cni.cncf.io/v1"
+kind: NetworkAttachmentDefinition
+metadata:
+  name: {{ nad_name }}
+  namespace: {{ name_space }}
+  labels:
+    tft-tests: "secondary-nad"
+spec:
+  config: '{
+    "cniVersion": "0.4.0",
+    "name": {{ nad_name }},
+    "netAttachDefName": {{ net_attach_def_name }},
+    "Role": "secondary",
+    "topology": {{ topology }},
+    "subnets": {{ subnets }},
+    "mtu": {{ mtu }},{% if has_resource_name %}
+    "resourceName": {{ resource_name }},{% endif %}
+    "type": "ovn-k8s-cni-overlay",
+    "logFile": "/var/log/ovn-kubernetes/flowtest.log",
+    "logLevel": "5",
+    "logfile-maxsize": 100,
+    "logfile-maxbackups": 5,
+    "logfile-maxage": 5
+  }'

tftbase.py

@@ -184,6 +184,10 @@ def get_tft_external_server_string() -> Optional[str]:
 ENV_TFT_UDN_PRIMARY_CIDR = "TFT_UDN_PRIMARY_CIDR"
 ENV_TFT_UDN_SECONDARY_CIDR = "TFT_UDN_SECONDARY_CIDR"
 
+ENV_TFT_SECONDARY_NAD_SUBNETS = "TFT_SECONDARY_NAD_SUBNETS"
+ENV_TFT_SECONDARY_NAD_MTU = "TFT_SECONDARY_NAD_MTU"
+ENV_TFT_SECONDARY_NAD_TOPOLOGY = "TFT_SECONDARY_NAD_TOPOLOGY"
+
 
 @functools.cache
 def get_udn_primary_cidr() -> str:
@@ -199,6 +203,27 @@ def get_udn_secondary_cidr() -> str:
     return s
 
 
+@functools.cache
+def get_secondary_nad_subnets() -> str:
+    s = get_environ(ENV_TFT_SECONDARY_NAD_SUBNETS) or "10.193.0.0/16/26"
+    logger.info(f"env: {ENV_TFT_SECONDARY_NAD_SUBNETS}={shlex.quote(s)}")
+    return s
+
+
+@functools.cache
+def get_secondary_nad_mtu() -> int:
+    s = get_environ(ENV_TFT_SECONDARY_NAD_MTU) or "1500"
+    logger.info(f"env: {ENV_TFT_SECONDARY_NAD_MTU}={shlex.quote(s)}")
+    return int(s)
+
+
+@functools.cache
+def get_secondary_nad_topology() -> str:
+    s = get_environ(ENV_TFT_SECONDARY_NAD_TOPOLOGY) or "layer3"
+    logger.info(f"env: {ENV_TFT_SECONDARY_NAD_TOPOLOGY}={shlex.quote(s)}")
+    return s
+
+
 T = typing.TypeVar("T")
 
 

trafficFlowTests.py

@@ -136,6 +136,70 @@ def _setup_udn(self, cfg_descr: ConfigDescriptor) -> None:
         self._configure_namespace(cfg_descr, namespace=udn_ns)
         self._udn_setup_done = True
 
+    def _setup_secondary_nad(self, cfg_descr: ConfigDescriptor) -> None:
+        tft = cfg_descr.get_tft()
+        if not any(
+            tc.info.connection_mode
+            in (
+                tftbase.ConnectionMode.MULTI_HOME,
+                tftbase.ConnectionMode.MNP_2ND_ALLOW,
+                tftbase.ConnectionMode.MNP_2ND_DENY,
+                tftbase.ConnectionMode.MNP_PRIMARY_DENY,
+            )
+            for tc in tft.test_cases
+        ) and not any(c.secondary_network_nad is not None for c in tft.connections):
+            return
+
+        namespace = tft.namespace
+        client = cfg_descr.tc.client_tenant
+        nad = cfg_descr.get_tft().connections[0].effective_secondary_network_nad
+        nad_name = nad.split("/")[-1]
+
+        existing = client.oc_get(
+            f"network-attachment-definition/{nad_name}",
+            namespace=namespace,
+            may_fail=True,
+        )
+        if existing is not None:
+            return
+
+        logger.info(f"Creating secondary NAD {nad} in namespace {namespace}")
+
+        resource_names = {c.resource_name for c in tft.connections}
+        resource_name = (
+            resource_names.pop()
+            if len(resource_names) == 1 and None not in resource_names
+            else None
+        )
+
+        _j = json.dumps
+        in_template = tftbase.get_manifest("secondary-nad.yaml.j2")
+        out_yaml = tftbase.get_manifest_renderpath("secondary-nad.yaml")
+        kjinja2.render_file(
+            in_template,
+            {
+                "nad_name": _j(nad_name),
+                "name_space": _j(namespace),
+                "net_attach_def_name": _j(nad),
+                "subnets": _j(tftbase.get_secondary_nad_subnets()),
+                "mtu": tftbase.get_secondary_nad_mtu(),
+                "topology": _j(tftbase.get_secondary_nad_topology()),
+                "has_resource_name": resource_name is not None,
+                "resource_name": _j(resource_name or ""),
+            },
+            out_file=out_yaml,
+        )
+        client.oc(f"apply -f {out_yaml}", die_on_error=True)
+
+    def _cleanup_secondary_nad(self, cfg_descr: ConfigDescriptor) -> None:
+        namespace = cfg_descr.get_tft().namespace
+        client = cfg_descr.tc.client_tenant
+        client.oc(
+            "delete network-attachment-definition -l tft-tests=secondary-nad",
+            namespace=namespace,
+            may_fail=True,
+        )
+
     def _cleanup_multi_network_policies(self, cfg_descr: ConfigDescriptor) -> None:
         namespace = cfg_descr.get_tft().namespace
         client = cfg_descr.tc.client_tenant
@@ -192,6 +256,8 @@ def _cleanup_previous_testspace(
             client.oc("delete pods -l tft-tests", namespace=namespace)
             client.oc("delete services -l tft-tests", namespace=namespace)
             self._cleanup_multi_network_policies(cfg_descr)
+            if force_cleanup:
+                self._cleanup_secondary_nad(cfg_descr)
 
             if self._udn_setup_done:
                 udn_ns = f"{namespace}-udn"
@@ -412,6 +478,7 @@ def test_run(
         ns_created = self._configure_namespace(cfg_descr)
         self._cleanup_stale_udn(cfg_descr)
         self._cleanup_previous_testspace(cfg_descr, force_cleanup=True)
+        self._setup_secondary_nad(cfg_descr)
 
         try:
             if test.pre_provision: