feat(cli): gate update-state and reset --package on operator version; clone NodeState before mutation; add chainsaw rejection coverage

Signed-off-by: Alex Yuskauskas <ayuskauskas@nvidia.com>

Author: ayuskauskas

k8s-tests/chainsaw/cli/update-state-reject/assert-skyhook-complete.yaml

@@ -0,0 +1,22 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: cli-update-state-reject-test
+status:
+  status: complete

k8s-tests/chainsaw/cli/update-state-reject/chainsaw-test.yaml

@@ -0,0 +1,93 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  name: cli-update-state-reject
+spec:
+  timeouts:
+    assert: 120s
+    exec: 90s
+  steps:
+  # Step 0: Reset state from previous runs
+  - name: reset-state
+    try:
+    - script:
+        timeout: 30s
+        content: |
+          ../skyhook-cli reset cli-update-state-reject-test --confirm 2>/dev/null || true
+          echo "State reset complete"
+
+  # Step 1: Create a Skyhook and wait for it to complete
+  - name: setup-skyhook
+    try:
+    - apply:
+        file: skyhook.yaml
+    - assert:
+        file: assert-skyhook-complete.yaml
+
+  # Step 2: update-state must reject a package that is not in the Skyhook spec
+  - name: reject-unknown-package
+    try:
+    - script:
+        timeout: 30s
+        content: |
+          set -e
+          out=$(../skyhook-cli update-state cli-update-state-reject-test nonexistent-pkg 1.0.0 apply complete --confirm 2>&1) && {
+            echo "expected non-zero exit, got success: $out"
+            exit 1
+          }
+          echo "$out" | grep -qi "not found in spec" || {
+            echo "expected error mentioning 'not found in spec', got: $out"
+            exit 1
+          }
+          echo "unknown package correctly rejected"
+
+  # Step 3: update-state must reject an invalid stage value
+  - name: reject-invalid-stage
+    try:
+    - script:
+        timeout: 30s
+        content: |
+          set -e
+          out=$(../skyhook-cli update-state cli-update-state-reject-test test-package 1.0.0 bogus-stage complete --confirm 2>&1) && {
+            echo "expected non-zero exit, got success: $out"
+            exit 1
+          }
+          echo "$out" | grep -qi "invalid stage" || {
+            echo "expected error mentioning 'invalid stage', got: $out"
+            exit 1
+          }
+          echo "invalid stage correctly rejected"
+
+  # Step 4: update-state must reject an invalid state value
+  - name: reject-invalid-state
+    try:
+    - script:
+        timeout: 30s
+        content: |
+          set -e
+          out=$(../skyhook-cli update-state cli-update-state-reject-test test-package 1.0.0 apply weird-state --confirm 2>&1) && {
+            echo "expected non-zero exit, got success: $out"
+            exit 1
+          }
+          echo "$out" | grep -qi "invalid state" || {
+            echo "expected error mentioning 'invalid state', got: $out"
+            exit 1
+          }
+          echo "invalid state correctly rejected"

k8s-tests/chainsaw/cli/update-state-reject/skyhook.yaml

@@ -0,0 +1,33 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+apiVersion: skyhook.nvidia.com/v1alpha1
+kind: Skyhook
+metadata:
+  name: cli-update-state-reject-test
+spec:
+  nodeSelectors:
+    matchLabels:
+      skyhook.nvidia.com/test-node: skyhooke2e
+  packages:
+    test-package:
+      version: "1.0.0"
+      image: ghcr.io/nvidia/skyhook-packages/shellscript
+      configMap:
+        apply.sh: |
+          #!/bin/bash
+          echo "update-state reject test apply"
+          sleep 2

operator/cmd/cli/app/reset.go

@@ -308,6 +308,14 @@ func runPackageReset(
 		return err
 	}
 
+	skyhook, err := utils.GetSkyhook(ctx, kubeClient.Dynamic(), skyhookName)
+	if err != nil {
+		return fmt.Errorf("fetching Skyhook %q: %w", skyhookName, err)
+	}
+	if err := checkNodeStateOperatorVersion(ctx, cmd, kubeClient, cliCtx, skyhook); err != nil {
+		return err
+	}
+
 	annotationKey := nodeStateAnnotationPrefix + skyhookName
 
 	type target struct {

operator/cmd/cli/app/reset_test.go

@@ -28,15 +28,19 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/spf13/cobra"
+	"github.com/stretchr/testify/mock"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/kubernetes/fake"
 	ktesting "k8s.io/client-go/testing"
 
 	"github.com/NVIDIA/nodewright/operator/api/v1alpha1"
 	"github.com/NVIDIA/nodewright/operator/internal/cli/client"
 	"github.com/NVIDIA/nodewright/operator/internal/cli/context"
+	mockdynamic "github.com/NVIDIA/nodewright/operator/internal/mocks/dynamic"
 )
 
 // Helper function to create a test node with nodeState annotation
@@ -355,16 +359,24 @@ var _ = Describe("Reset Command", func() {
 
 	Describe("--package", func() {
 		var (
-			fakeKube   *fake.Clientset
-			kubeClient *client.Client
-			cliCtx     *context.CLIContext
-			cmd        *cobra.Command
-			out        *bytes.Buffer
+			fakeKube    *fake.Clientset
+			mockDynamic *mockdynamic.Interface
+			kubeClient  *client.Client
+			cliCtx      *context.CLIContext
+			cmd         *cobra.Command
+			out         *bytes.Buffer
 		)
 
+		skyhookGVR := schema.GroupVersionResource{
+			Group:    "skyhook.nvidia.com",
+			Version:  "v1alpha1",
+			Resource: "skyhooks",
+		}
+
 		BeforeEach(func() {
 			fakeKube = fake.NewClientset()
-			kubeClient = client.NewWithClientsAndConfig(fakeKube, nil, nil)
+			mockDynamic = &mockdynamic.Interface{}
+			kubeClient = client.NewWithClientsAndConfig(fakeKube, mockDynamic, nil)
 			cliCtx = context.NewCLIContext(nil)
 			cmd = NewResetCmd(cliCtx)
 			out = &bytes.Buffer{}
@@ -373,6 +385,22 @@ var _ = Describe("Reset Command", func() {
 			cmd.SetIn(bytes.NewBufferString("y\n"))
 		})
 
+		setupSkyhookCR := func(name, version string) {
+			sk := &v1alpha1.Skyhook{}
+			sk.Name = name
+			sk.Annotations = map[string]string{
+				"skyhook.nvidia.com/version": version,
+			}
+			u := &unstructured.Unstructured{}
+			raw, err := json.Marshal(sk)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(json.Unmarshal(raw, &u.Object)).To(Succeed())
+			u.SetGroupVersionKind(schema.GroupVersionKind{Group: "skyhook.nvidia.com", Version: "v1alpha1", Kind: "Skyhook"})
+			mockNSRes := &mockdynamic.NamespaceableResourceInterface{}
+			mockDynamic.On("Resource", skyhookGVR).Return(mockNSRes)
+			mockNSRes.On("Get", mock.Anything, name, mock.Anything, mock.Anything).Return(u, nil)
+		}
+
 		addNode := func(name string, ns v1alpha1.NodeState) {
 			raw, _ := json.Marshal(ns)
 			n := &corev1.Node{
@@ -388,6 +416,7 @@ var _ = Describe("Reset Command", func() {
 		}
 
 		It("removes only the named package entry from each node", func() {
+			setupSkyhookCR("demo", "v0.15.0")
 			addNode("n1", v1alpha1.NodeState{
 				"pkg1|1.0": {Name: "pkg1", Version: "1.0", Stage: v1alpha1.StageApply, State: v1alpha1.StateComplete},
 				"pkg2|2.0": {Name: "pkg2", Version: "2.0", Stage: v1alpha1.StageConfig, State: v1alpha1.StateComplete},
@@ -405,6 +434,7 @@ var _ = Describe("Reset Command", func() {
 		})
 
 		It("with name:version only removes if version matches", func() {
+			setupSkyhookCR("demo", "v0.15.0")
 			addNode("n1", v1alpha1.NodeState{
 				"pkg1|1.0": {Name: "pkg1", Version: "1.0", Stage: v1alpha1.StageApply, State: v1alpha1.StateComplete},
 			})
@@ -420,6 +450,7 @@ var _ = Describe("Reset Command", func() {
 		})
 
 		It("removes the entire annotation when the last entry is removed", func() {
+			setupSkyhookCR("demo", "v0.15.0")
 			addNode("n1", v1alpha1.NodeState{
 				"pkg1|1.0": {Name: "pkg1", Version: "1.0", Stage: v1alpha1.StageApply, State: v1alpha1.StateComplete},
 			})
@@ -440,5 +471,18 @@ var _ = Describe("Reset Command", func() {
 			Entry("empty name with version", ":1.0"),
 			Entry("empty version with colon", "pkg1:"),
 		)
+
+		It("rejects --package against an operator older than v0.15.0", func() {
+			setupSkyhookCR("demo", "v0.10.0")
+			addNode("n1", v1alpha1.NodeState{
+				"pkg1|1.0": {Name: "pkg1", Version: "1.0", Stage: v1alpha1.StageApply, State: v1alpha1.StateComplete},
+			})
+			opts := &resetOptions{confirm: true, skipBatchReset: true, pkg: "pkg1"}
+			err := runReset(gocontext.Background(), cmd, kubeClient, "demo", opts, cliCtx)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("does not support"))
+			Expect(err.Error()).To(ContainSubstring("v0.10.0"))
+			Expect(err.Error()).To(ContainSubstring("v0.15.0"))
+		})
 	})
 })

operator/cmd/cli/app/update_state.go

@@ -22,6 +22,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"maps"
 	"slices"
 	"sort"
 
@@ -169,7 +170,10 @@ func buildUpdateStateTarget(
 	addMode bool,
 ) (updateStateTarget, bool) {
 	uniqueName := pkg.GetUniqueName()
-	ns := nodeStates[name]
+	// why: nodeStates[name] returns a shared map reference; mutating it would
+	// silently mutate the caller's snapshot. Clone before mutation so this
+	// helper stays safe even if it's ever called twice for the same node.
+	ns := maps.Clone(nodeStates[name])
 	if ns == nil {
 		ns = v1alpha1.NodeState{}
 	}
@@ -222,6 +226,11 @@ func runUpdateState(ctx context.Context, cmd *cobra.Command, kubeClient *client.
 	if err != nil {
 		return fmt.Errorf("fetching Skyhook %q: %w", skyhookName, err)
 	}
+
+	if err := checkNodeStateOperatorVersion(ctx, cmd, kubeClient, cliCtx, skyhook); err != nil {
+		return err
+	}
+
 	pkg, ok := skyhook.Spec.Packages[packageName]
 	if !ok || pkg.Version != packageVersion {
 		return fmt.Errorf("package %q (version %q) not found in spec of Skyhook %q", packageName, packageVersion, skyhookName)
@@ -310,3 +319,32 @@ func runUpdateState(ctx context.Context, cmd *cobra.Command, kubeClient *client.
 	_, _ = fmt.Fprintf(cmd.OutOrStdout(), "\nSuccessfully updated %d node(s)\n", success)
 	return firstErr
 }
+
+// checkNodeStateOperatorVersion rejects the call when the running operator is
+// older than MinNodeStateSupportVersion. The check first reads the version
+// annotation written by the operator onto the Skyhook CR; when that's missing
+// or non-semver (e.g. "dev") it falls back to inspecting the operator
+// Deployment. If neither source yields a valid version we warn but allow the
+// edit to proceed — better than refusing every command in clusters where the
+// CLI can't see the operator's namespace.
+func checkNodeStateOperatorVersion(
+	ctx context.Context,
+	cmd *cobra.Command,
+	kubeClient *client.Client,
+	cliCtx *cliContext.CLIContext,
+	skyhook *v1alpha1.Skyhook,
+) error {
+	opVersion := utils.GetSkyhookVersion(skyhook)
+	if opVersion == "" || !utils.IsValidVersion(opVersion) {
+		deployVersion, derr := utils.DiscoverOperatorVersion(ctx, kubeClient.Kubernetes(), cliCtx.GlobalFlags.Namespace())
+		if derr == nil && utils.IsValidVersion(deployVersion) {
+			opVersion = deployVersion
+		} else {
+			_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: unable to determine operator version; cannot verify compatibility (requires %s+)\n", utils.MinNodeStateSupportVersion)
+		}
+	}
+	if utils.IsValidVersion(opVersion) && utils.CompareVersions(opVersion, utils.MinNodeStateSupportVersion) < 0 {
+		return fmt.Errorf("operator version %s does not support this command; minimum supported version is %s", opVersion, utils.MinNodeStateSupportVersion)
+	}
+	return nil
+}

operator/cmd/cli/app/update_state_test.go

@@ -261,6 +261,30 @@ var _ = Describe("UpdateState Command", func() {
 			Expect(err.Error()).To(ContainSubstring("apiserver unreachable"))
 		})
 
+		It("rejects update-state against an operator older than v0.15.0", func() {
+			sk := &v1alpha1.Skyhook{}
+			sk.Name = "demo"
+			sk.Annotations = map[string]string{"skyhook.nvidia.com/version": "v0.10.0"}
+			sk.Spec.Packages = v1alpha1.Packages{
+				"pkg1": {PackageRef: v1alpha1.PackageRef{Name: "pkg1", Version: "1.0"}, Image: "example.com/pkg1"},
+			}
+			u := &unstructured.Unstructured{}
+			raw, err := json.Marshal(sk)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(json.Unmarshal(raw, &u.Object)).To(Succeed())
+			u.SetGroupVersionKind(schema.GroupVersionKind{Group: "skyhook.nvidia.com", Version: "v1alpha1", Kind: "Skyhook"})
+			mockNSRes := &mockdynamic.NamespaceableResourceInterface{}
+			mockDynamic.On("Resource", skyhookGVR).Return(mockNSRes)
+			mockNSRes.On("Get", mock.Anything, "demo", mock.Anything, mock.Anything).Return(u, nil)
+
+			opts := &updateStateOptions{confirm: true}
+			err = runUpdateState(gocontext.Background(), cmd, kubeClient, []string{"demo", "pkg1", "1.0", "config", "complete"}, opts, cliCtx)
+			Expect(err).To(HaveOccurred())
+			Expect(err.Error()).To(ContainSubstring("does not support"))
+			Expect(err.Error()).To(ContainSubstring("v0.10.0"))
+			Expect(err.Error()).To(ContainSubstring("v0.15.0"))
+		})
+
 		Describe("--add", func() {
 			It("requires --node or --selector", func() {
 				setupSkyhookCR(map[string]string{"pkg1": "1.0"})

operator/internal/cli/utils/utils.go

@@ -327,6 +327,11 @@ const (
 	DefaultNamespace = "skyhook"
 	// MinAnnotationSupportVersion is the minimum operator version that supports annotation-based pause/disable
 	MinAnnotationSupportVersion = "v0.8.0"
+	// MinNodeStateSupportVersion is the lowest operator version known to use the
+	// current map[string]PackageStatus shape for the
+	// skyhook.nvidia.com/nodeState_<skyhook> annotation. Commands that edit that
+	// annotation refuse to run against older operators.
+	MinNodeStateSupportVersion = "v0.15.0"
 )
 
 // CompareVersions compares two semver versions.