diff --git a/.github/workflows/e2e-kernel-mode.yml b/.github/workflows/e2e-kernel-mode.yml new file mode 100644 index 0000000000..6e78b9c9d9 --- /dev/null +++ b/.github/workflows/e2e-kernel-mode.yml @@ -0,0 +1,96 @@ +name: E2E Kernel Mode +on: + push: + branches: [main] + pull_request: + branches: [main] +jobs: + e2e-kernel: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version-file: go.mod + + - name: Write kind config + run: | + cat > /tmp/kind-config.yaml <<'EOF' + kind: Cluster + apiVersion: kind.x-k8s.io/v1alpha4 + containerdConfigPatches: + - |- + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:5000"] + endpoint = ["http://kind-registry:5000"] + nodes: + - role: control-plane + - role: worker + - role: worker + EOF + + - name: Start local registry + run: | + docker run -d --restart=always -p 5000:5000 --name kind-registry registry:2 + + - name: Create kind cluster + uses: helm/kind-action@v1 + with: + cluster_name: kernel-mode-test + node_image: kindest/node:v1.32.0 + config: /tmp/kind-config.yaml + + - name: Connect registry to kind network + run: | + docker network connect kind kind-registry || true + + - name: Create secondary docker networks and attach to kind nodes + run: | + docker network create --driver bridge kind-secondary-1 + docker network create --driver bridge kind-secondary-2 + for node in $(kind get nodes --name kernel-mode-test); do + docker network connect kind-secondary-1 "$node" + docker network connect kind-secondary-2 "$node" + done + + - name: Setup kind nodes (kernel modules, OVS) + run: | + for node in $(kind get nodes --name kernel-mode-test); do + docker exec "$node" bash -c "modprobe 8021q && modprobe bonding && modprobe openvswitch" + docker exec "$node" bash -c "apt-get update -qq && apt-get install -y -qq openvswitch-switch > /dev/null 2>&1 && systemctl start openvswitch-switch" + done + + - name: Label kind worker nodes + run: | + for node in $(kubectl get nodes --no-headers -o custom-columns=NAME:.metadata.name | grep worker); do + kubectl label node "$node" node-role.kubernetes.io/worker="" --overwrite + done + + - name: Deploy with cluster-sync + run: | + make cluster-up + make cluster-sync + env: + KUBEVIRT_PROVIDER: external + KUBECONFIG: /home/runner/.kube/config + DEV_IMAGE_REGISTRY: localhost:5000 + IMAGE_BUILDER: docker + KUBEVIRT_NUM_NODES: 3 + HANDLER_EXTRA_PARAMS: "--build-arg NMSTATE_SOURCE=packit" + + - name: Run kernel mode e2e tests + run: make test-e2e-handler-kernel + env: + KUBEVIRT_PROVIDER: external + KUBECONFIG: /home/runner/.kube/config + SSH: /bin/true + PRIMARY_NIC: eth0 + FIRST_SECONDARY_NIC: eth1 + SECOND_SECONDARY_NIC: eth2 + + - name: Upload test logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: kernel-mode-test-logs + path: test_logs/ + if-no-files-found: ignore diff --git a/Makefile b/Makefile index 4b739c0181..0cff274537 100644 --- a/Makefile +++ b/Makefile @@ -205,9 +205,19 @@ test-reporter: go run . --dry-run --fake=failure && \ go run . --dry-run --fake=stale +# Features supported in kernel mode (no NetworkManager). +# NodeSSH excluded: requires SSH to nodes (nmcli). +# VLAN creation works but assigning static IP on VLAN fails (nispor limitation). +KERNEL_MODE_FEATURES ?= (Nodes && !NodeSSH) || NNSDependencies || NNSFilter || \ + (NNSTimestamp && !NodeSSH) + test-e2e-handler: KUBECONFIG=$(KUBECONFIG) OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE) MONITORING_NAMESPACE=$(MONITORING_NAMESPACE) $(GINKGO) $(e2e_test_args) ./test/e2e/handler ... +test-e2e-handler-kernel: + KUBECONFIG=$(KUBECONFIG) OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE) MONITORING_NAMESPACE=$(MONITORING_NAMESPACE) \ + $(GINKGO) $(e2e_test_args) --label-filter="$(KERNEL_MODE_FEATURES)" ./test/e2e/handler ... + test-e2e-operator: manifests KUBECONFIG=$(KUBECONFIG) OPERATOR_NAMESPACE=$(OPERATOR_NAMESPACE) MONITORING_NAMESPACE=$(MONITORING_NAMESPACE) $(GINKGO) $(e2e_test_args) ./test/e2e/operator ... @@ -286,6 +296,7 @@ olm-push: bundle-push index-push check-gen \ operator-sdk \ test-e2e-handler \ + test-e2e-handler-kernel \ test-e2e-operator \ test-e2e \ test-reporter\ diff --git a/build/install-nmstate.packit.sh b/build/install-nmstate.packit.sh new file mode 100755 index 0000000000..92af92840c --- /dev/null +++ b/build/install-nmstate.packit.sh @@ -0,0 +1,5 @@ +#!/bin/bash -xe + +dnf install -b -y dnf-plugins-core +dnf copr enable -y packit/nmstate-nmstate-3104 +dnf install -b -y nmstate diff --git a/cluster/up.sh b/cluster/up.sh index 44adbffe31..110a6d7555 100755 --- a/cluster/up.sh +++ b/cluster/up.sh @@ -4,6 +4,7 @@ set -ex source ./cluster/kubevirtci.sh kubevirtci::install +source ./cluster/sync-common.sh $(kubevirtci::path)/cluster-up/up.sh @@ -11,19 +12,23 @@ if [[ "$KUBEVIRT_PROVIDER" =~ ^(okd|ocp)-.*$$ ]]; then \ while ! $(KUBECTL) get securitycontextconstraints; do sleep 1; done; \ fi -echo 'Upgrading NetworkManager and enabling and starting up openvswitch' -for node in $(./cluster/kubectl.sh get nodes --no-headers | awk '{print $1}'); do - if [[ "$NM_VERSION" == "latest" ]]; then - echo "Installing NetworkManager from copr networkmanager/NetworkManager-main" - ./cluster/cli.sh ssh ${node} -- sudo dnf install -y dnf-plugins-core - ./cluster/cli.sh ssh ${node} -- sudo dnf copr enable -y networkmanager/NetworkManager-main - fi - ./cluster/cli.sh ssh ${node} -- sudo dnf upgrade -y NetworkManager --allowerasing - ./cluster/cli.sh ssh ${node} -- sudo systemctl daemon-reload - ./cluster/cli.sh ssh ${node} -- sudo systemctl enable openvswitch - ./cluster/cli.sh ssh ${node} -- sudo systemctl restart openvswitch - # Newer kubevirtci has dhclient installed so we should enforce not using it to - # keep using the NM internal DHCP client as we always have - ./cluster/cli.sh ssh ${node} -- sudo rm -f /etc/NetworkManager/conf.d/002-dhclient.conf - ./cluster/cli.sh ssh ${node} -- sudo systemctl restart NetworkManager -done +if isExternal; then + echo 'Skipping NetworkManager/OVS setup for external provider' +else + echo 'Upgrading NetworkManager and enabling and starting up openvswitch' + for node in $(./cluster/kubectl.sh get nodes --no-headers | awk '{print $1}'); do + if [[ "$NM_VERSION" == "latest" ]]; then + echo "Installing NetworkManager from copr networkmanager/NetworkManager-main" + ./cluster/cli.sh ssh ${node} -- sudo dnf install -y dnf-plugins-core + ./cluster/cli.sh ssh ${node} -- sudo dnf copr enable -y networkmanager/NetworkManager-main + fi + ./cluster/cli.sh ssh ${node} -- sudo dnf upgrade -y NetworkManager --allowerasing + ./cluster/cli.sh ssh ${node} -- sudo systemctl daemon-reload + ./cluster/cli.sh ssh ${node} -- sudo systemctl enable openvswitch + ./cluster/cli.sh ssh ${node} -- sudo systemctl restart openvswitch + # Newer kubevirtci has dhclient installed so we should enforce not using it to + # keep using the NM internal DHCP client as we always have + ./cluster/cli.sh ssh ${node} -- sudo rm -f /etc/NetworkManager/conf.d/002-dhclient.conf + ./cluster/cli.sh ssh ${node} -- sudo systemctl restart NetworkManager + done +fi diff --git a/cmd/handler/main.go b/cmd/handler/main.go index fe8ab0bd9c..f0e0ed16c2 100644 --- a/cmd/handler/main.go +++ b/cmd/handler/main.go @@ -63,6 +63,7 @@ import ( "github.com/nmstate/kubernetes-nmstate/pkg/file" nmstatelog "github.com/nmstate/kubernetes-nmstate/pkg/log" "github.com/nmstate/kubernetes-nmstate/pkg/monitoring" + "github.com/nmstate/kubernetes-nmstate/pkg/nm" "github.com/nmstate/kubernetes-nmstate/pkg/nmstatectl" "github.com/nmstate/kubernetes-nmstate/pkg/webhook" ) @@ -224,9 +225,29 @@ func setupWebhookEnvironment(mgr manager.Manager) error { return nil } +// detectAndSetKernelMode checks if NetworkManager is available and enables kernel mode if not. +func detectAndSetKernelMode() { + _, err := nm.Version() + if err != nil { + setupLog.Info("NetworkManager not available, enabling kernel mode", + "error", err.Error()) + nmstatectl.SetKernelMode(true) + } else { + setupLog.Info("NetworkManager detected, using standard mode") + } +} + // setupHandlerEnvironment cleans up unavailableNodeCounts after unexpected restart, // configures the handler controllers and performs health checks func setupHandlerEnvironment(mgr manager.Manager) error { + detectAndSetKernelMode() + + if nmstatectl.IsKernelMode() { + if err := file.Touch("/tmp/kernel-mode"); err != nil { + setupLog.Error(err, "Failed to write kernel-mode flag file") + } + } + // Clean stale unavailable counts from node before starting controllers // Prevents deadlock after unexpected cluster reboot where nodes were // processing NNCP and left stale counts in etcd. diff --git a/controllers/handler/node_controller.go b/controllers/handler/node_controller.go index a8be5171f5..3e2eafe85a 100644 --- a/controllers/handler/node_controller.go +++ b/controllers/handler/node_controller.go @@ -128,9 +128,14 @@ func (r *NodeReconciler) getDependencyVersions() *nmstate.DependencyVersions { r.Log.Error(err, "failed retrieving handler nmstate version") } - hostNetworkManagerVersion, err := nm.Version() - if err != nil { - r.Log.Error(err, "error retrieving host Networkmanager version") + var hostNetworkManagerVersion string + if nmstatectl.IsKernelMode() { + hostNetworkManagerVersion = "N/A (kernel mode)" + } else { + hostNetworkManagerVersion, err = nm.Version() + if err != nil { + r.Log.Error(err, "error retrieving host Networkmanager version") + } } return &nmstate.DependencyVersions{ diff --git a/controllers/operator/nmstate_controller_test.go b/controllers/operator/nmstate_controller_test.go index 4c5b59399c..f4f4bc5f2d 100644 --- a/controllers/operator/nmstate_controller_test.go +++ b/controllers/operator/nmstate_controller_test.go @@ -487,7 +487,7 @@ var _ = Describe("NMState controller reconcile", func() { ds := &appsv1.DaemonSet{} err := cl.Get(context.Background(), handlerKey, ds) Expect(err).ToNot(HaveOccurred()) - expectedCommand := "nmstatectl show -vv 2>&1" + expectedCommand := "if [ -f /tmp/kernel-mode ]; then nmstatectl show -k -vv 2>&1; else nmstatectl show -vv 2>&1; fi" Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.Exec.Command).To(ContainElement(expectedCommand)) }) }) @@ -515,7 +515,7 @@ var _ = Describe("NMState controller reconcile", func() { ds := &appsv1.DaemonSet{} err := cl.Get(context.Background(), handlerKey, ds) Expect(err).ToNot(HaveOccurred()) - expectedCommand := "nmstatectl show 2>&1" + expectedCommand := "if [ -f /tmp/kernel-mode ]; then nmstatectl show -k 2>&1; else nmstatectl show 2>&1; fi" Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.Exec.Command).To(ContainElement(expectedCommand)) }) }) @@ -542,7 +542,7 @@ var _ = Describe("NMState controller reconcile", func() { ds := &appsv1.DaemonSet{} err := cl.Get(context.Background(), handlerKey, ds) Expect(err).ToNot(HaveOccurred()) - expectedCommand := "nmstatectl show 2>&1" + expectedCommand := "if [ -f /tmp/kernel-mode ]; then nmstatectl show -k 2>&1; else nmstatectl show 2>&1; fi" Expect(ds.Spec.Template.Spec.Containers[0].LivenessProbe.Exec.Command).To(ContainElement(expectedCommand)) }) }) diff --git a/deploy/handler/operator.yaml b/deploy/handler/operator.yaml index 81d8878584..2c815f7c90 100644 --- a/deploy/handler/operator.yaml +++ b/deploy/handler/operator.yaml @@ -361,6 +361,19 @@ spec: tolerations: {{ toYaml .HandlerTolerations | nindent 8 }} affinity: {{ toYaml .HandlerAffinity | nindent 8 }} priorityClassName: system-node-critical + initContainers: + - name: ensure-dbus-socket + image: {{ .HandlerImage }} + imagePullPolicy: {{ .HandlerPullPolicy }} + command: + - sh + - -c + - "mkdir -p /host-run/dbus && test -e /host-run/dbus/system_bus_socket || touch /host-run/dbus/system_bus_socket" + volumeMounts: + - name: host-run + mountPath: /host-run + securityContext: + privileged: true containers: - name: nmstate-handler args: @@ -445,17 +458,19 @@ spec: command: - bash - -c - - "nmstatectl show {{ .HandlerReadinessProbeExtraArg }} 2>&1" + - "if [ -f /tmp/kernel-mode ]; then nmstatectl show -k {{ .HandlerReadinessProbeExtraArg }} 2>&1; else nmstatectl show {{ .HandlerReadinessProbeExtraArg }} 2>&1; fi" initialDelaySeconds: 60 periodSeconds: 60 timeoutSeconds: 10 successThreshold: 1 failureThreshold: 5 volumes: + - name: host-run + hostPath: + path: /run - name: dbus-socket hostPath: path: /run/dbus/system_bus_socket - type: Socket - name: nmstate-lock hostPath: path: /var/k8s_nmstate diff --git a/pkg/client/client.go b/pkg/client/client.go index 46d9dde292..bab3bd2a82 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -154,6 +154,15 @@ func ApplyDesiredState(ctx context.Context, cli client.Client, desiredState shar return "Ignoring empty desired state", nil } + if nmstatectl.IsKernelMode() { + log.Info("Kernel mode: applying desired state without checkpoint/rollback/probes") + setOutput, err := nmstatectl.Set(desiredState, DesiredStateConfigurationTimeout) + if err != nil { + return setOutput, err + } + return fmt.Sprintf("setOutput: %s \n", setOutput), nil + } + // Before apply we get the probes that are working fine, they should be // working fine after apply probes := probe.Select(ctx, cli) diff --git a/pkg/nmstatectl/nmstatectl.go b/pkg/nmstatectl/nmstatectl.go index 25de1bd662..e4e45c20fa 100644 --- a/pkg/nmstatectl/nmstatectl.go +++ b/pkg/nmstatectl/nmstatectl.go @@ -28,12 +28,20 @@ import ( "time" "github.com/pkg/errors" + logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/yaml" nmstate "github.com/nmstate/kubernetes-nmstate/api/shared" ) -var debugMode bool +var ( + debugMode bool + kernelMode bool + log = logf.Log.WithName("nmstatectl") +) + +func SetKernelMode(kernel bool) { kernelMode = kernel } +func IsKernelMode() bool { return kernelMode } const nmstateCommand = "nmstatectl" @@ -88,11 +96,20 @@ func nmstatectl(arguments []string) (string, error) { } func ShowWithArgumentsAndOutputs(arguments []string, stdout, stderr io.Writer) error { - return nmstatectlWithInputAndOutputs(append([]string{"show"}, arguments...), "", stdout, stderr) + args := []string{"show"} + if kernelMode { + args = append(args, "-k") + } + args = append(args, arguments...) + return nmstatectlWithInputAndOutputs(args, "", stdout, stderr) } func Show() (string, error) { - return nmstatectl([]string{"show"}) + args := []string{"show"} + if kernelMode { + args = append(args, "-k") + } + return nmstatectl(args) } func Set(desiredState nmstate.State, timeout time.Duration) (string, error) { @@ -103,17 +120,30 @@ func Set(desiredState nmstate.State, timeout time.Duration) (string, error) { if debugMode { args = append(args, "-vv") } - args = append(args, "--no-commit", "--timeout", strconv.Itoa(int(timeout.Seconds()))) + if kernelMode { + log.Info("Kernel mode: applying with -k flag, skipping --no-commit and --timeout (checkpoints not supported)") + args = append(args, "-k") + } else { + args = append(args, "--no-commit", "--timeout", strconv.Itoa(int(timeout.Seconds()))) + } setOutput, err := nmstatectlWithInput(args, string(desiredState.Raw)) return setOutput, err } func Commit() (string, error) { + if kernelMode { + log.Info("Kernel mode: skipping commit (checkpoints not supported)") + return "commit skipped (kernel mode)", nil + } return nmstatectl([]string{"commit"}) } func Rollback() error { + if kernelMode { + log.Info("Kernel mode: skipping rollback (checkpoints not supported)") + return nil + } _, err := nmstatectl([]string{"rollback"}) if err != nil { return errors.Wrapf(err, "failed calling nmstatectl rollback") @@ -136,6 +166,10 @@ func NewStats(features []string) *Stats { } func Statistic(desiredState nmstate.State) (*Stats, error) { + if kernelMode { + log.Info("Kernel mode: skipping statistics (not supported without NetworkManager)") + return NewStats(nil), nil + } statsOutput, err := nmstatectlWithInput( []string{"st", "-"}, string(desiredState.Raw), diff --git a/pkg/nmstatectl/nmstatectl_test.go b/pkg/nmstatectl/nmstatectl_test.go index 83a3eb6de5..c453201b38 100644 --- a/pkg/nmstatectl/nmstatectl_test.go +++ b/pkg/nmstatectl/nmstatectl_test.go @@ -51,9 +51,11 @@ func TestSetCommandAndDebugMode(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { originalDebugMode := debugMode + originalKernelMode := kernelMode originalExecCommand := execCommand defer func() { debugMode = originalDebugMode + kernelMode = originalKernelMode execCommand = originalExecCommand }() @@ -71,6 +73,7 @@ func TestSetCommandAndDebugMode(t *testing.T) { desiredState := nmstate.State{Raw: []byte(`{"interfaces": []}`)} SetDebugMode(tt.debugMode) + SetKernelMode(false) _, err := Set(desiredState, timeout) if err != nil { t.Errorf("Set() failed: %v", err) @@ -86,3 +89,154 @@ func TestSetCommandAndDebugMode(t *testing.T) { }) } } + +func TestSetKernelMode(t *testing.T) { + tests := []struct { + name string + debugMode bool + kernelMode bool + expectedArgs []string + }{ + { + name: "kernel mode without debug", + debugMode: false, + kernelMode: true, + expectedArgs: []string{"apply", "-k"}, + }, + { + name: "kernel mode with debug", + debugMode: true, + kernelMode: true, + expectedArgs: []string{"apply", "-vv", "-k"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + originalDebugMode := debugMode + originalKernelMode := kernelMode + originalExecCommand := execCommand + defer func() { + debugMode = originalDebugMode + kernelMode = originalKernelMode + execCommand = originalExecCommand + }() + + var capturedArgs []string + execCommand = func(name string, args ...string) *exec.Cmd { + capturedArgs = args + return exec.CommandContext(context.TODO(), "echo", "mocked output") + } + + SetDebugMode(tt.debugMode) + SetKernelMode(tt.kernelMode) + + desiredState := nmstate.State{Raw: []byte(`{"interfaces": []}`)} + _, err := Set(desiredState, 120*time.Second) + if err != nil { + t.Errorf("Set() failed: %v", err) + } + + if !reflect.DeepEqual(capturedArgs, tt.expectedArgs) { + t.Errorf("Arguments = %v, want %v", capturedArgs, tt.expectedArgs) + } + }) + } +} + +func TestShowKernelMode(t *testing.T) { + tests := []struct { + name string + kernelMode bool + expectedArgs []string + }{ + { + name: "show without kernel mode", + kernelMode: false, + expectedArgs: []string{"show"}, + }, + { + name: "show with kernel mode", + kernelMode: true, + expectedArgs: []string{"show", "-k"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + originalKernelMode := kernelMode + originalExecCommand := execCommand + defer func() { + kernelMode = originalKernelMode + execCommand = originalExecCommand + }() + + var capturedArgs []string + execCommand = func(name string, args ...string) *exec.Cmd { + capturedArgs = args + return exec.CommandContext(context.TODO(), "echo", "mocked output") + } + + SetKernelMode(tt.kernelMode) + _, err := Show() + if err != nil { + t.Errorf("Show() failed: %v", err) + } + + if !reflect.DeepEqual(capturedArgs, tt.expectedArgs) { + t.Errorf("Arguments = %v, want %v", capturedArgs, tt.expectedArgs) + } + }) + } +} + +func TestCommitKernelMode(t *testing.T) { + originalKernelMode := kernelMode + originalExecCommand := execCommand + defer func() { + kernelMode = originalKernelMode + execCommand = originalExecCommand + }() + + commandExecuted := false + execCommand = func(name string, args ...string) *exec.Cmd { + commandExecuted = true + return exec.CommandContext(context.TODO(), "echo", "mocked output") + } + + SetKernelMode(true) + output, err := Commit() + if err != nil { + t.Errorf("Commit() failed: %v", err) + } + if commandExecuted { + t.Error("Commit() should not execute a command in kernel mode") + } + if output != "commit skipped (kernel mode)" { + t.Errorf("Commit() output = %q, want %q", output, "commit skipped (kernel mode)") + } +} + +func TestRollbackKernelMode(t *testing.T) { + originalKernelMode := kernelMode + originalExecCommand := execCommand + defer func() { + kernelMode = originalKernelMode + execCommand = originalExecCommand + }() + + commandExecuted := false + execCommand = func(name string, args ...string) *exec.Cmd { + commandExecuted = true + return exec.CommandContext(context.TODO(), "echo", "mocked output") + } + + SetKernelMode(true) + err := Rollback() + if err != nil { + t.Errorf("Rollback() failed: %v", err) + } + if commandExecuted { + t.Error("Rollback() should not execute a command in kernel mode") + } +} diff --git a/test/e2e/handler/bonding_default_interface_test.go b/test/e2e/handler/bonding_default_interface_test.go index 28b35bba14..69942837f4 100644 --- a/test/e2e/handler/bonding_default_interface_test.go +++ b/test/e2e/handler/bonding_default_interface_test.go @@ -62,7 +62,7 @@ func bondAbsentWithPrimaryUp(bondName string) nmstate.State { `, bondName, primaryNic)) } -var _ = Describe("NodeNetworkConfigurationPolicy bonding default interface", func() { +var _ = Describe("NodeNetworkConfigurationPolicy bonding default interface", Label("DHCP"), func() { Context("when there is a default interface with dynamic address", func() { addressByNode := map[string]string{} BeforeEach(func() { diff --git a/test/e2e/handler/default_bridged_network_test.go b/test/e2e/handler/default_bridged_network_test.go index cb6c9b79ad..80f3a398de 100644 --- a/test/e2e/handler/default_bridged_network_test.go +++ b/test/e2e/handler/default_bridged_network_test.go @@ -64,7 +64,7 @@ func resetDefaultInterface() nmstate.State { `, primaryNic)) } -var _ = Describe("NodeNetworkConfigurationPolicy default bridged network", func() { +var _ = Describe("NodeNetworkConfigurationPolicy default bridged network", Label("DHCP"), func() { var ( DefaultNetwork = "default-network" ) diff --git a/test/e2e/handler/default_bridged_network_with_nmpolicy_test.go b/test/e2e/handler/default_bridged_network_with_nmpolicy_test.go index 383ebcf85d..e97948a08c 100644 --- a/test/e2e/handler/default_bridged_network_with_nmpolicy_test.go +++ b/test/e2e/handler/default_bridged_network_with_nmpolicy_test.go @@ -28,7 +28,7 @@ import ( "github.com/nmstate/kubernetes-nmstate/test/e2e/policy" ) -var _ = Describe("NodeNetworkConfigurationPolicy default bridged network with nmpolicy", func() { +var _ = Describe("NodeNetworkConfigurationPolicy default bridged network with nmpolicy", Label("DHCP"), func() { var ( DefaultNetwork = "default-network" ) diff --git a/test/e2e/handler/default_ovs_bridged_network_test.go b/test/e2e/handler/default_ovs_bridged_network_test.go index b6382a2187..b7aeeb7669 100644 --- a/test/e2e/handler/default_ovs_bridged_network_test.go +++ b/test/e2e/handler/default_ovs_bridged_network_test.go @@ -66,7 +66,7 @@ func ovsBridgeWithTheDefaultInterfaceAbsent(ovsBridgeName, ovsBridgeInternalPort `, primaryNic, ovsBridgeInternalPortName, ovsBridgeName)) } -var _ = Describe("NodeNetworkConfigurationPolicy default ovs-bridged network", func() { +var _ = Describe("NodeNetworkConfigurationPolicy default ovs-bridged network", Label("OVS"), func() { Context("when there is a default interface with dynamic address", func() { const ( ovsDefaultNetwork = "ovs-default-network" diff --git a/test/e2e/handler/dns_test.go b/test/e2e/handler/dns_test.go index edb37ef7b7..5b12934994 100644 --- a/test/e2e/handler/dns_test.go +++ b/test/e2e/handler/dns_test.go @@ -178,7 +178,7 @@ func emptyGlobalDNSConfig() nmstate.State { `) } -var _ = Describe("Dns configuration", func() { +var _ = Describe("Dns configuration", Label("DHCP"), func() { Context("when desiredState is configured", func() { var ( searchDomain1 = "fufu.ostest.test.metalkube.org" diff --git a/test/e2e/handler/examples_test.go b/test/e2e/handler/examples_test.go index 4b901fead8..bcd950bbd7 100644 --- a/test/e2e/handler/examples_test.go +++ b/test/e2e/handler/examples_test.go @@ -30,7 +30,7 @@ import ( // It only checks the top level API, hence it does not verify that the // configuration is indeed applied on nodes. That should be tested by dedicated // test suites for each feature. -var _ = Describe("[user-guide] Examples", func() { +var _ = Describe("[user-guide] Examples", Label("Examples"), func() { beforeTestIfaceExample := func(fileName string) { kubectlAndCheck("apply", "-f", fmt.Sprintf("docs/examples/%s", fileName)) diff --git a/test/e2e/handler/lldp_with_nmpolicy_test.go b/test/e2e/handler/lldp_with_nmpolicy_test.go index c0551d060a..c2e047cd17 100644 --- a/test/e2e/handler/lldp_with_nmpolicy_test.go +++ b/test/e2e/handler/lldp_with_nmpolicy_test.go @@ -33,7 +33,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/client" ) -var _ = Describe("LLDP configuration with nmpolicy", func() { +var _ = Describe("LLDP configuration with nmpolicy", Label("LLDP"), func() { var lldpdPod *corev1.Pod lldpEnabledPolicyName := "lldp-enabled" diff --git a/test/e2e/handler/main_test.go b/test/e2e/handler/main_test.go index 66fba52e80..cac52c1a72 100644 --- a/test/e2e/handler/main_test.go +++ b/test/e2e/handler/main_test.go @@ -29,11 +29,13 @@ import ( . "github.com/onsi/gomega" corev1 "k8s.io/api/core/v1" + k8stypes "k8s.io/apimachinery/pkg/types" "sigs.k8s.io/controller-runtime/pkg/client" logf "sigs.k8s.io/controller-runtime/pkg/log" "sigs.k8s.io/controller-runtime/pkg/log/zap" + nmstatev1beta1 "github.com/nmstate/kubernetes-nmstate/api/v1beta1" testenv "github.com/nmstate/kubernetes-nmstate/test/env" "github.com/nmstate/kubernetes-nmstate/test/environment" knmstatereporter "github.com/nmstate/kubernetes-nmstate/test/reporter" @@ -51,6 +53,7 @@ var ( dnsTestNic string portFieldName string miimonFormat string + isKernelMode bool nodesInitialInterfacesState = make(map[string]map[string]string) interfacesToIgnore = []string{"flannel.1", "dummy0", "tunl0"} knmstateReporter *knmstatereporter.KubernetesNMStateReporter @@ -93,16 +96,26 @@ var _ = BeforeSuite(func() { } } - resetDesiredStateForAllNodes() - expectedInitialState := interfacesState(resetPrimaryAndSecondaryNICs(), interfacesToIgnore) - for _, node := range allNodes { - Eventually(func(g Gomega) { - By("Wait for network configuration to show up at NNS to retrieve it") - nodeState := nodeInterfacesState(node, interfacesToIgnore) - for name, state := range expectedInitialState { - g.Expect(nodeState).To(HaveKeyWithValue(name, state)) - } - }, 20*time.Second, time.Second).Should(Succeed()) + By("Detecting kernel mode from NNS") + nns := nmstatev1beta1.NodeNetworkState{} + err = testenv.Client.Get(context.TODO(), k8stypes.NamespacedName{Name: allNodes[0]}, &nns) + Expect(err).ToNot(HaveOccurred()) + isKernelMode = nns.Status.HostNetworkManagerVersion == "N/A (kernel mode)" + + if isKernelMode { + By("Kernel mode detected — skipping NIC reset (nmstatectl apply -k cannot modify interfaces)") + } else { + resetDesiredStateForAllNodes() + expectedInitialState := interfacesState(resetPrimaryAndSecondaryNICs(), interfacesToIgnore) + for _, node := range allNodes { + Eventually(func(g Gomega) { + By("Wait for network configuration to show up at NNS to retrieve it") + nodeState := nodeInterfacesState(node, interfacesToIgnore) + for name, state := range expectedInitialState { + g.Expect(nodeState).To(HaveKeyWithValue(name, state)) + } + }, 20*time.Second, time.Second).Should(Succeed()) + } } knmstateReporter = knmstatereporter.New("test_logs/e2e/handler", testenv.OperatorNamespace, nodes) knmstateReporter.Cleanup() diff --git a/test/e2e/handler/metrics_test.go b/test/e2e/handler/metrics_test.go index 0f596ec8dc..b3430f7923 100644 --- a/test/e2e/handler/metrics_test.go +++ b/test/e2e/handler/metrics_test.go @@ -29,7 +29,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("Metrics", func() { +var _ = Describe("Metrics", Label("Metrics"), func() { var ( extraBridgeName = func() string { return bridge1 + "-extra" } linuxBridgeWithCustomHostname = func(bridge string) nmstate.State { diff --git a/test/e2e/handler/multiple_policies_for_same_node_test.go b/test/e2e/handler/multiple_policies_for_same_node_test.go index ddb3ea1e5e..67f7f7f373 100644 --- a/test/e2e/handler/multiple_policies_for_same_node_test.go +++ b/test/e2e/handler/multiple_policies_for_same_node_test.go @@ -26,7 +26,7 @@ import ( "github.com/nmstate/kubernetes-nmstate/test/e2e/policy" ) -var _ = Describe("NodeNetworkState", func() { +var _ = Describe("NodeNetworkState", Label("MultiplePolicies"), func() { Context("with multiple policies configured", func() { var ( vlanIDs = []string{"102", "103"} diff --git a/test/e2e/handler/nnce_conditions_test.go b/test/e2e/handler/nnce_conditions_test.go index 1b0967ed20..9cf86c026e 100644 --- a/test/e2e/handler/nnce_conditions_test.go +++ b/test/e2e/handler/nnce_conditions_test.go @@ -29,7 +29,7 @@ import ( policyconditions "github.com/nmstate/kubernetes-nmstate/test/e2e/policy" ) -var _ = Describe("EnactmentCondition", func() { +var _ = Describe("EnactmentCondition", Label("EnactmentCondition"), func() { Context("when applying valid config", func() { AfterEach(func() { By("Remove the bridge") diff --git a/test/e2e/handler/nnce_desiredstate_test.go b/test/e2e/handler/nnce_desiredstate_test.go index 0178d84bdd..be70deab62 100644 --- a/test/e2e/handler/nnce_desiredstate_test.go +++ b/test/e2e/handler/nnce_desiredstate_test.go @@ -25,7 +25,7 @@ import ( "github.com/nmstate/kubernetes-nmstate/test/e2e/policy" ) -var _ = Describe("Enactment DesiredState", func() { +var _ = Describe("Enactment DesiredState", Label("EnactmentDesiredState"), func() { Context("when applying a policy to matching nodes", func() { BeforeEach(func() { By("Create a policy") diff --git a/test/e2e/handler/nncp_cleanup_test.go b/test/e2e/handler/nncp_cleanup_test.go index 33b97cc40e..073844833d 100644 --- a/test/e2e/handler/nncp_cleanup_test.go +++ b/test/e2e/handler/nncp_cleanup_test.go @@ -31,7 +31,7 @@ import ( testenv "github.com/nmstate/kubernetes-nmstate/test/env" ) -var _ = Describe("NNCP cleanup", func() { +var _ = Describe("NNCP cleanup", Label("PolicyCleanup"), func() { BeforeEach(func() { By("Create a policy") diff --git a/test/e2e/handler/nncp_parallel_test.go b/test/e2e/handler/nncp_parallel_test.go index 811a142475..961d91887f 100644 --- a/test/e2e/handler/nncp_parallel_test.go +++ b/test/e2e/handler/nncp_parallel_test.go @@ -42,7 +42,7 @@ func enactmentsFailingOrProgressing(policy string) int { return failingOrProgressingEnactments } -var _ = Describe("NNCP with maxUnavailable", func() { +var _ = Describe("NNCP with maxUnavailable", Label("PolicyParallel"), func() { duration := 15 * time.Second interval := 500 * time.Millisecond Context("when applying a policy to matching nodes", func() { diff --git a/test/e2e/handler/nns_dependencies_test.go b/test/e2e/handler/nns_dependencies_test.go index fa95c28c1e..bbe9a2278d 100644 --- a/test/e2e/handler/nns_dependencies_test.go +++ b/test/e2e/handler/nns_dependencies_test.go @@ -24,7 +24,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -var _ = Describe("[nns] NNS Dependencies", func() { +var _ = Describe("[nns] NNS Dependencies", Label("NNSDependencies"), func() { BeforeEach(func() { // Make sure NNSes are present for _, node := range nodes { diff --git a/test/e2e/handler/nns_filter_test.go b/test/e2e/handler/nns_filter_test.go index cbf112d3bc..a2c6c271ac 100644 --- a/test/e2e/handler/nns_filter_test.go +++ b/test/e2e/handler/nns_filter_test.go @@ -26,7 +26,7 @@ import ( "k8s.io/apimachinery/pkg/types" ) -var _ = Describe("[nns] NNS Interface filter", func() { +var _ = Describe("[nns] NNS Interface filter", Label("NNSFilter"), func() { BeforeEach(func() { // Make sure NNSes are present for _, node := range nodes { diff --git a/test/e2e/handler/nns_ovn.go b/test/e2e/handler/nns_ovn.go index 584f253836..22b3b6ab09 100644 --- a/test/e2e/handler/nns_ovn.go +++ b/test/e2e/handler/nns_ovn.go @@ -27,7 +27,7 @@ import ( policyconditions "github.com/nmstate/kubernetes-nmstate/test/e2e/policy" ) -var _ = Describe("[nns] NNS OVN bridge mappings", func() { +var _ = Describe("[nns] NNS OVN bridge mappings", Label("OVN"), func() { const ( bridgeName = "ovsbr1" networkName = "net1" diff --git a/test/e2e/handler/nns_update_timestamp_test.go b/test/e2e/handler/nns_update_timestamp_test.go index 65e789a06f..5607413bc2 100644 --- a/test/e2e/handler/nns_update_timestamp_test.go +++ b/test/e2e/handler/nns_update_timestamp_test.go @@ -33,7 +33,7 @@ import ( const expectedDummyName = "dummy0" -var _ = Describe("[nns] NNS LastSuccessfulUpdateTime", func() { +var _ = Describe("[nns] NNS LastSuccessfulUpdateTime", Label("NNSTimestamp"), func() { var ( originalNNSs map[string]nmstatev1beta1.NodeNetworkState ) @@ -65,7 +65,7 @@ var _ = Describe("[nns] NNS LastSuccessfulUpdateTime", func() { }, timeout, interval).Should(Succeed()) }) }) - Context("when network configuration is changed externally", func() { + Context("when network configuration is changed externally", Label("NodeSSH"), func() { BeforeEach(func() { createDummyConnectionAtAllNodes(expectedDummyName) }) diff --git a/test/e2e/handler/node_selector_test.go b/test/e2e/handler/node_selector_test.go index ebb38862d0..a710599cbd 100644 --- a/test/e2e/handler/node_selector_test.go +++ b/test/e2e/handler/node_selector_test.go @@ -34,7 +34,7 @@ import ( "github.com/nmstate/kubernetes-nmstate/pkg/enactment" ) -var _ = Describe("NodeSelector", func() { +var _ = Describe("NodeSelector", Label("NodeSelector"), func() { var ( testNodeSelector = map[string]string{"testKey": "testValue"} numberOfEnactmentsForPolicy = func(policyName string) int { diff --git a/test/e2e/handler/nodes_test.go b/test/e2e/handler/nodes_test.go index 5330ffad1b..dbe380ead8 100644 --- a/test/e2e/handler/nodes_test.go +++ b/test/e2e/handler/nodes_test.go @@ -26,7 +26,7 @@ import ( "github.com/nmstate/kubernetes-nmstate/pkg/node" ) -var _ = Describe("Nodes", func() { +var _ = Describe("Nodes", Label("Nodes"), func() { Context("when are up", func() { It("should have NodeNetworkState with currentState for each node", func() { for _, node := range nodes { @@ -43,7 +43,7 @@ var _ = Describe("Nodes", func() { } }) }) - Context("and new interface is configured", func() { + Context("and new interface is configured", Label("NodeSSH"), func() { expectedDummyName := "dummy0" BeforeEach(func() { diff --git a/test/e2e/handler/pending_checkpoint_test.go b/test/e2e/handler/pending_checkpoint_test.go index f063548d0e..b4047ff1ed 100644 --- a/test/e2e/handler/pending_checkpoint_test.go +++ b/test/e2e/handler/pending_checkpoint_test.go @@ -27,7 +27,7 @@ import ( "github.com/nmstate/kubernetes-nmstate/test/runner" ) -var _ = Describe("checkpoin", func() { +var _ = Describe("checkpoin", Label("Checkpoint"), func() { Context("when is not committed from previous operation", func() { BeforeEach(func() { stateAsJSON, err := linuxBrUpNoPorts(bridge1).MarshalJSON() diff --git a/test/e2e/handler/rollback_test.go b/test/e2e/handler/rollback_test.go index e1ab9947b0..5a08dac59f 100644 --- a/test/e2e/handler/rollback_test.go +++ b/test/e2e/handler/rollback_test.go @@ -128,7 +128,7 @@ interfaces: `, nic)) } -var _ = Describe("rollback", func() { +var _ = Describe("rollback", Label("Rollback"), func() { // This spec is done only at first node since policy has to be different // per node (ip addresses has to be different at cluster). Context("when connectivity to default gw is lost after state configuration", func() { diff --git a/test/e2e/handler/simple_bridge_and_bond_test.go b/test/e2e/handler/simple_bridge_and_bond_test.go index fe43d549d9..4fd7418b5b 100644 --- a/test/e2e/handler/simple_bridge_and_bond_test.go +++ b/test/e2e/handler/simple_bridge_and_bond_test.go @@ -127,7 +127,7 @@ func bondUpWithEth1Eth2AndVlan(bondName string) nmstate.State { `, bondName, fmt.Sprintf(miimonFormat, 140), portFieldName, firstSecondaryNic, secondSecondaryNic, bondName, bondName)) } -var _ = Describe("NodeNetworkState", func() { +var _ = Describe("NodeNetworkState", Label("DHCP"), func() { Context("when desiredState is configured", func() { Context("with a linux bridge up with no ports", func() { BeforeEach(func() { diff --git a/test/e2e/handler/simple_ovs_bridge_and_bond_test.go b/test/e2e/handler/simple_ovs_bridge_and_bond_test.go index d9c302b40a..679c112322 100644 --- a/test/e2e/handler/simple_ovs_bridge_and_bond_test.go +++ b/test/e2e/handler/simple_ovs_bridge_and_bond_test.go @@ -115,7 +115,7 @@ func ovsBrAndInternalPortAbsent(bridgeName, internalPortName string) nmstate.Sta `, internalPortName, bridgeName)) } -var _ = Describe("OVS Bridge", func() { +var _ = Describe("OVS Bridge", Label("OVS"), func() { Context("when desiredState is updated with ovs-bridge with link aggregation port", func() { verifyInterfaces := func() { for _, node := range nodes { diff --git a/test/e2e/handler/simple_ovs_bridge_test.go b/test/e2e/handler/simple_ovs_bridge_test.go index 6b2175da8b..73aa846e3c 100644 --- a/test/e2e/handler/simple_ovs_bridge_test.go +++ b/test/e2e/handler/simple_ovs_bridge_test.go @@ -22,7 +22,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("Simple OVS bridge", func() { +var _ = Describe("Simple OVS bridge", Label("OVS"), func() { Context("when desiredState is configured with an ovs bridge up", func() { BeforeEach(func() { updateDesiredStateAndWait(ovsBrUp(bridge1)) diff --git a/test/e2e/handler/simple_vlan_and_ip_test.go b/test/e2e/handler/simple_vlan_and_ip_test.go index 2d49d4f282..4373ab9b3d 100644 --- a/test/e2e/handler/simple_vlan_and_ip_test.go +++ b/test/e2e/handler/simple_vlan_and_ip_test.go @@ -24,7 +24,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("NodeNetworkState", func() { +var _ = Describe("NodeNetworkState", Label("VLAN"), func() { Context("when vlan configured", func() { var ( vlanID = "102" diff --git a/test/e2e/handler/states.go b/test/e2e/handler/states.go index 340ef752d0..dc635a4812 100644 --- a/test/e2e/handler/states.go +++ b/test/e2e/handler/states.go @@ -316,6 +316,33 @@ func resetPrimaryAndSecondaryNICs() nmstate.State { `, primaryNic, firstSecondaryNic, secondSecondaryNic)) } +func resetPrimaryAndSecondaryNICsKernelMode() nmstate.State { + return nmstate.NewState(fmt.Sprintf(`interfaces: + - name: %s + type: ethernet + state: up + ipv4: + enabled: true + ipv6: + enabled: true + - name: %s + type: ethernet + state: up + ipv4: + enabled: false + ipv6: + enabled: false + - name: %s + state: up + type: ethernet + ipv4: + enabled: false + ipv6: + enabled: false + +`, primaryNic, firstSecondaryNic, secondSecondaryNic)) +} + func bridgeOnTheSecondaryInterfaceState() nmstate.State { return nmstate.NewState(`interfaces: - name: brext diff --git a/test/e2e/handler/static_addr_and_route_test.go b/test/e2e/handler/static_addr_and_route_test.go index f145875b44..fe3fdab810 100644 --- a/test/e2e/handler/static_addr_and_route_test.go +++ b/test/e2e/handler/static_addr_and_route_test.go @@ -147,7 +147,7 @@ routes: `, firstSecondaryNic, firstSecondaryNic)) } -var _ = Describe("Static addresses and routes", func() { +var _ = Describe("Static addresses and routes", Label("StaticAddress"), func() { Context("when desiredState is configured", func() { var ( node string diff --git a/test/e2e/handler/upgrade_test.go b/test/e2e/handler/upgrade_test.go index 82091662cb..4d4b935eb3 100644 --- a/test/e2e/handler/upgrade_test.go +++ b/test/e2e/handler/upgrade_test.go @@ -33,7 +33,7 @@ import ( testenv "github.com/nmstate/kubernetes-nmstate/test/env" ) -var _ = Describe("NodeNetworkConfigurationPolicy upgrade", func() { +var _ = Describe("NodeNetworkConfigurationPolicy upgrade", Label("Upgrade"), func() { Context("when v1beta1 is populated", func() { BeforeEach(func() { // Ensure TestPolicy doesn't exist before creating v1beta1 policy diff --git a/test/e2e/handler/user_guide_test.go b/test/e2e/handler/user_guide_test.go index fb4684342a..8c4f1b6f05 100644 --- a/test/e2e/handler/user_guide_test.go +++ b/test/e2e/handler/user_guide_test.go @@ -25,7 +25,7 @@ import ( "github.com/nmstate/kubernetes-nmstate/test/e2e/policy" ) -var _ = Describe("[user-guide] Introduction", func() { +var _ = Describe("[user-guide] Introduction", Label("UserGuide"), func() { runConfiguration := func() { kubectlAndCheck("apply", "-f", "docs/user-guide/bond0-eth1-eth2_up.yaml") kubectlAndCheck("wait", "nncp", "bond0-eth1-eth2", "--for", "condition=Available", "--timeout", "4m") diff --git a/test/e2e/handler/utils.go b/test/e2e/handler/utils.go index 9da04fb631..379b0e56dc 100644 --- a/test/e2e/handler/utils.go +++ b/test/e2e/handler/utils.go @@ -247,16 +247,24 @@ func updateDesiredStateWithCaptureAtNodeAndWait(node string, desiredState nmstat // TODO: After we implement policy delete (it will cleanUp desiredState) we have to remove this. func resetDesiredStateForNodes() { + resetNICsState := resetPrimaryAndSecondaryNICs() + if isKernelMode { + resetNICsState = resetPrimaryAndSecondaryNICsKernelMode() + } By("Resetting nics state primary up and secondaries disable ipv4 and ipv6") - updateDesiredState(resetPrimaryAndSecondaryNICs()) + updateDesiredState(resetNICsState) defer deletePolicy(TestPolicy) policy.WaitForAvailableTestPolicy() } // TODO: After we implement policy delete (it will cleanUp desiredState) we have to remove this. func resetDesiredStateForAllNodes() { + resetNICsState := resetPrimaryAndSecondaryNICs() + if isKernelMode { + resetNICsState = resetPrimaryAndSecondaryNICsKernelMode() + } By("Resetting nics state primary up and secondaries disable ipv4 and ipv6 at all nodes") - setDesiredStateWithPolicyWithoutNodeSelector(TestPolicy, resetPrimaryAndSecondaryNICs()) + setDesiredStateWithPolicyWithoutNodeSelector(TestPolicy, resetNICsState) defer deletePolicy(TestPolicy) policy.WaitForAvailableTestPolicy() } diff --git a/test/e2e/handler/vrf_test.go b/test/e2e/handler/vrf_test.go index 48e10c84d8..8416235beb 100644 --- a/test/e2e/handler/vrf_test.go +++ b/test/e2e/handler/vrf_test.go @@ -23,7 +23,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("NodeNetworkState", func() { +var _ = Describe("NodeNetworkState", Label("VRF"), func() { var ( node string ) diff --git a/test/e2e/handler/webhook_test.go b/test/e2e/handler/webhook_test.go index 13a5588d41..e84c4767e3 100644 --- a/test/e2e/handler/webhook_test.go +++ b/test/e2e/handler/webhook_test.go @@ -35,7 +35,7 @@ import ( // We just check the labe at CREATE/UPDATE events since mutated data is already // check at unit test. -var _ = Describe("Mutating Admission Webhook", func() { +var _ = Describe("Mutating Admission Webhook", Label("Webhook"), func() { Context("when policy is created", func() { BeforeEach(func() { // Make sure test policy is not there so @@ -77,7 +77,7 @@ var _ = Describe("Mutating Admission Webhook", func() { }) }) -var _ = Describe("Validation Admission Webhook", func() { +var _ = Describe("Validation Admission Webhook", Label("Webhook"), func() { Context("When a policy is created and progressing", func() { BeforeEach(func() { By("Creating a policy without waiting for it to be available")