feat: add cli doc for backwards compatibly and warnings

Author: lockwobr

docs/cli.md

@@ -6,6 +6,50 @@ kubectl plugin for managing Skyhook deployments, packages, and nodes.
 
 The Skyhook CLI (`kubectl skyhook`) provides SRE tooling for managing Skyhook operators and their packages across Kubernetes cluster nodes. It supports inspecting node/package state, forcing re-runs, managing node lifecycle, and retrieving logs.
 
+## Compatibility
+
+### Minimum Operator Version
+
+The CLI requires **operator version v0.8.0 or later** for full functionality of all commands.
+
+### Command Compatibility Matrix
+
+| Command | v0.7.x and earlier | v0.8.0+ |
+|---------|-------------------|---------|
+| `version` | ✅ Full | ✅ Full |
+| `node status` | ✅ Full | ✅ Full |
+| `node list` | ✅ Full | ✅ Full |
+| `node reset` | ✅ Full | ✅ Full |
+| `node ignore/unignore` | ✅ Full | ✅ Full |
+| `package status` | ✅ Full | ✅ Full |
+| `package rerun` | ✅ Full | ✅ Full |
+| `package logs` | ✅ Full | ✅ Full |
+| `reset` | ✅ Full | ✅ Full |
+| `pause` | ❌ Not supported | ✅ Full |
+| `resume` | ❌ Not supported | ✅ Full |
+| `disable` | ❌ Not supported | ✅ Full |
+| `enable` | ❌ Not supported | ✅ Full |
+
+### Breaking Change: Pause/Disable Mechanism
+
+In operator versions **v0.7.x and earlier**, pausing and disabling a Skyhook was done via spec fields:
+
+```yaml
+spec:
+  pause: true  # Old method - no longer used by operator
+```
+
+Starting with **v0.8.0**, the operator uses **annotations** instead:
+
+```yaml
+metadata:
+  annotations:
+    skyhook.nvidia.com/pause: "true"
+    skyhook.nvidia.com/disable: "true"
+```
+
+The CLI's `pause`, `resume`, `disable`, and `enable` commands set these annotations. If you're running an older operator (v0.7.x or earlier), these commands will appear to succeed but the operator won't recognize the annotations - you'll need to edit the Skyhook spec directly using `kubectl edit`.
+
 ## Installation
 
 ```bash
@@ -56,6 +100,8 @@ kubectl skyhook version --timeout 10s
 
 Control Skyhook processing state.
 
+> **Note:** Requires operator v0.8.0+. See [Compatibility](#compatibility) for details.
+
 ```bash
 # Pause a Skyhook (stops processing new nodes)
 kubectl skyhook pause my-skyhook
@@ -69,6 +115,8 @@ kubectl skyhook resume my-skyhook
 
 Completely disable or re-enable a Skyhook.
 
+> **Note:** Requires operator v0.8.0+. See [Compatibility](#compatibility) for details.
+
 ```bash
 # Disable a Skyhook completely
 kubectl skyhook disable my-skyhook
@@ -205,6 +253,9 @@ kubectl skyhook node status --skyhook my-skyhook -o json
 ```
 
 ### Emergency Stop
+
+> **Note:** Requires operator v0.8.0+. For older operators, use `kubectl edit skyhook my-skyhook` and set `spec.pause: true`.
+
 ```bash
 # Pause all processing
 kubectl skyhook pause my-skyhook --confirm

operator/Makefile

@@ -334,11 +334,10 @@ ensure-test-symlinks:  ## Ensure symlinks exist in test directories (idempotent)
 	done
 
 .PHONY: build-cli
-CLI_LDFLAGS := -ldflags "-X github.com/NVIDIA/skyhook/operator/internal/version.VERSION=$(VERSION) -X github.com/NVIDIA/skyhook/operator/internal/version.GIT_SHA=$(GIT_SHA)"
+CLI_VERSION ?= $(CLI_TAG_LAST)
+CLI_LDFLAGS := -ldflags "-X github.com/NVIDIA/skyhook/operator/internal/version.VERSION=$(CLI_VERSION) -X github.com/NVIDIA/skyhook/operator/internal/version.GIT_SHA=$(GIT_SHA)"
 build-cli: ensure-test-symlinks ## Build CLI binary.
 	go build $(GOFLAGS) $(CLI_LDFLAGS) -o bin/skyhook cmd/cli/main.go
-
-CLI_VERSION ?= $(CLI_TAG_LAST)
 CLI_DIST_DIR ?= dist
 cli-release-build: ## Build CLI binaries for all platforms (used by CI release workflow).
 	@echo "Building CLI $(CLI_VERSION) for all platforms..."

operator/cmd/cli/app/cli_test.go

@@ -24,6 +24,7 @@ import (
 	"testing"
 
 	"github.com/NVIDIA/skyhook/operator/internal/cli/context"
+	"github.com/NVIDIA/skyhook/operator/internal/cli/utils"
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	"github.com/spf13/cobra"
@@ -177,34 +178,34 @@ var _ = Describe("Skyhook CLI Tests", func() {
 		})
 	})
 
-	Describe("extractImageTag", func() {
+	Describe("ExtractImageTag", func() {
 		It("should extract tag from simple image reference", func() {
-			tag := extractImageTag("ghcr.io/nvidia/skyhook/operator:v1.2.3")
+			tag := utils.ExtractImageTag("ghcr.io/nvidia/skyhook/operator:v1.2.3")
 			Expect(tag).To(Equal("v1.2.3"))
 		})
 
 		It("should extract tag from image with digest", func() {
-			tag := extractImageTag("ghcr.io/nvidia/skyhook/operator:v1.2.3@sha256:abc123")
+			tag := utils.ExtractImageTag("ghcr.io/nvidia/skyhook/operator:v1.2.3@sha256:abc123")
 			Expect(tag).To(Equal("v1.2.3"))
 		})
 
 		It("should return empty for image without tag", func() {
-			tag := extractImageTag("ghcr.io/nvidia/skyhook/operator")
+			tag := utils.ExtractImageTag("ghcr.io/nvidia/skyhook/operator")
 			Expect(tag).To(BeEmpty())
 		})
 
 		It("should handle image with only digest", func() {
-			tag := extractImageTag("ghcr.io/nvidia/skyhook/operator@sha256:abc123")
+			tag := utils.ExtractImageTag("ghcr.io/nvidia/skyhook/operator@sha256:abc123")
 			Expect(tag).To(BeEmpty())
 		})
 
 		It("should extract latest tag", func() {
-			tag := extractImageTag("nginx:latest")
+			tag := utils.ExtractImageTag("nginx:latest")
 			Expect(tag).To(Equal("latest"))
 		})
 
 		It("should handle image with port in registry", func() {
-			tag := extractImageTag("localhost:5000/myimage:v1.0")
+			tag := utils.ExtractImageTag("localhost:5000/myimage:v1.0")
 			Expect(tag).To(Equal("v1.0"))
 		})
 	})

operator/cmd/cli/app/lifecycle.go

@@ -89,6 +89,42 @@ func newLifecycleCmd(ctx *cliContext.CLIContext, cfg lifecycleConfig) *cobra.Com
 				return fmt.Errorf("initializing kubernetes client: %w", err)
 			}
 
+			// Fetch the Skyhook to check operator version from its annotation
+			skyhook, err := utils.GetSkyhook(cmd.Context(), kubeClient.Dynamic(), skyhookName)
+			if err != nil {
+				return err
+			}
+
+			// Check operator version compatibility using Skyhook's version annotation
+			// If annotation is missing or invalid (e.g., "dev"), fall back to deployment version
+			opVersion := utils.GetSkyhookVersion(skyhook)
+			if opVersion == "" || !utils.IsValidVersion(opVersion) {
+				// Try to get version from deployment instead
+				deployVersion, err := utils.DiscoverOperatorVersion(cmd.Context(), kubeClient.Kubernetes(), ctx.GlobalFlags.Namespace())
+				if err == nil && utils.IsValidVersion(deployVersion) {
+					opVersion = deployVersion
+				} else {
+					_, _ = fmt.Fprintf(cmd.ErrOrStderr(), "Warning: unable to determine operator version; "+
+						"cannot verify compatibility (requires %s+)\n", utils.MinAnnotationSupportVersion)
+				}
+			}
+
+			if utils.IsValidVersion(opVersion) && utils.CompareVersions(opVersion, utils.MinAnnotationSupportVersion) < 0 {
+				// Operator is older than v0.8.0 - annotations won't work
+				if cfg.annotation == utils.PauseAnnotation {
+					// pause/resume - feature exists but uses spec field instead of annotation
+					specValue := "true"
+					if cfg.action == "remove" {
+						specValue = "false"
+					}
+					return fmt.Errorf("operator version %s uses spec.pause instead of annotations; "+
+						"use 'kubectl edit skyhook %s' and set spec.pause: %s", opVersion, skyhookName, specValue)
+				}
+				// disable/enable - feature doesn't exist at all in older versions
+				return fmt.Errorf("operator version %s does not support %s; this feature was added in %s",
+					opVersion, cfg.confirmVerb, utils.MinAnnotationSupportVersion)
+			}
+
 			if cfg.action == "set" {
 				err = utils.SetSkyhookAnnotation(cmd.Context(), kubeClient.Dynamic(), skyhookName, cfg.annotation, "true")
 			} else {
@@ -141,12 +177,15 @@ func NewResumeCmd(ctx *cliContext.CLIContext) *cobra.Command {
 
 The operator will resume processing nodes after this command.`,
 		example: `  # Resume a paused Skyhook
-  kubectl skyhook resume gpu-init`,
+  kubectl skyhook resume gpu-init
+
+  # Resume without confirmation
+  kubectl skyhook resume gpu-init --confirm`,
 		annotation:   utils.PauseAnnotation,
 		action:       "remove",
 		verb:         "resumed",
 		confirmVerb:  "resume",
-		needsConfirm: false,
+		needsConfirm: true,
 	})
 }
 
@@ -181,11 +220,14 @@ func NewEnableCmd(ctx *cliContext.CLIContext) *cobra.Command {
 
 The operator will resume normal processing after this command.`,
 		example: `  # Enable a disabled Skyhook
-  kubectl skyhook enable gpu-init`,
+  kubectl skyhook enable gpu-init
+
+  # Enable without confirmation
+  kubectl skyhook enable gpu-init --confirm`,
 		annotation:   utils.DisableAnnotation,
 		action:       "remove",
 		verb:         "enabled",
 		confirmVerb:  "enable",
-		needsConfirm: false,
+		needsConfirm: true,
 	})
 }

operator/cmd/cli/app/lifecycle_test.go

@@ -49,7 +49,7 @@ var _ = Describe("Lifecycle Commands", func() {
 			cmdFactory:     NewResumeCmd,
 			expectedUse:    "resume <skyhook-name>",
 			expectedVerb:   "Resume",
-			hasConfirmFlag: false,
+			hasConfirmFlag: true,
 		},
 		{
 			name:           "Disable",
@@ -63,7 +63,7 @@ var _ = Describe("Lifecycle Commands", func() {
 			cmdFactory:     NewEnableCmd,
 			expectedUse:    "enable <skyhook-name>",
 			expectedVerb:   "Enable",
-			hasConfirmFlag: false,
+			hasConfirmFlag: true,
 		},
 	}
 

operator/cmd/cli/app/version.go

@@ -21,16 +21,13 @@ package app
 import (
 	"context"
 	"fmt"
-	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
-	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/client-go/kubernetes"
 
 	"github.com/NVIDIA/skyhook/operator/internal/cli/client"
 	cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
+	"github.com/NVIDIA/skyhook/operator/internal/cli/utils"
 	"github.com/NVIDIA/skyhook/operator/internal/version"
 )
 
@@ -72,7 +69,7 @@ func NewVersionCmd(ctx *cliContext.CLIContext) *cobra.Command {
 			cmdCtx, cancel := context.WithTimeout(cmd.Context(), timeout)
 			defer cancel()
 
-			opVersion, err := discoverOperatorVersion(cmdCtx, kubeClient.Kubernetes(), ctx.GlobalFlags.Namespace())
+			opVersion, err := utils.DiscoverOperatorVersion(cmdCtx, kubeClient.Kubernetes(), ctx.GlobalFlags.Namespace())
 			if err != nil {
 				_, _ = fmt.Fprintf(cmd.OutOrStdout(), "Skyhook operator:\tunknown (%v)\n", err)
 				return nil
@@ -88,56 +85,3 @@ func NewVersionCmd(ctx *cliContext.CLIContext) *cobra.Command {
 
 	return versionCmd
 }
-
-func discoverOperatorVersion(ctx context.Context, kube kubernetes.Interface, namespace string) (string, error) {
-	if kube == nil {
-		return "", fmt.Errorf("nil kubernetes client")
-	}
-	if namespace == "" {
-		namespace = cliContext.DefaultNamespace
-	}
-
-	deploymentName := "skyhook-operator-controller-manager"
-	deployment, err := kube.AppsV1().Deployments(namespace).Get(ctx, deploymentName, metav1.GetOptions{})
-	if err != nil {
-		if errors.IsNotFound(err) {
-			return "", fmt.Errorf("skyhook operator deployment %q not found in namespace %q", deploymentName, namespace)
-		}
-		return "", fmt.Errorf("querying operator deployment: %w", err)
-	}
-
-	// Try to get version from Helm label (preferred for Helm deployments)
-	if v := deployment.Labels["app.kubernetes.io/version"]; strings.TrimSpace(v) != "" {
-		return strings.TrimSpace(v), nil
-	}
-
-	// Fallback: parse version from container image tag (works for kustomize deployments)
-	if len(deployment.Spec.Template.Spec.Containers) > 0 {
-		image := deployment.Spec.Template.Spec.Containers[0].Image
-		if tag := extractImageTag(image); tag != "" && tag != "latest" {
-			return tag, nil
-		}
-	}
-
-	return "", fmt.Errorf("unable to determine operator version from deployment labels or image tag")
-}
-
-// extractImageTag extracts the tag from a container image reference.
-// Examples:
-//   - "ghcr.io/nvidia/skyhook/operator:v1.2.3" -> "v1.2.3"
-//   - "ghcr.io/nvidia/skyhook/operator:v1.2.3@sha256:..." -> "v1.2.3"
-//   - "ghcr.io/nvidia/skyhook/operator" -> ""
-func extractImageTag(image string) string {
-	// Remove digest if present (e.g., @sha256:...)
-	if idx := strings.Index(image, "@"); idx > 0 {
-		image = image[:idx]
-	}
-
-	// Split on ":" to get tag
-	parts := strings.Split(image, ":")
-	if len(parts) < 2 {
-		return ""
-	}
-
-	return strings.TrimSpace(parts[len(parts)-1])
-}

operator/cmd/cli/app/version_test.go

@@ -30,6 +30,7 @@ import (
 	"k8s.io/client-go/kubernetes/fake"
 
 	cliContext "github.com/NVIDIA/skyhook/operator/internal/cli/context"
+	"github.com/NVIDIA/skyhook/operator/internal/cli/utils"
 )
 
 // TestNewVersionCmd_ClientOnly verifies the version command works with --client-only flag
@@ -88,6 +89,7 @@ func TestDiscoverOperatorVersion(t *testing.T) {
 					Namespace: "skyhook",
 					Labels: map[string]string{
 						"app.kubernetes.io/version": "v0.9.0",
+						"app.kubernetes.io/name":    "skyhook-operator", // needed for detection
 					},
 				},
 			},
@@ -146,10 +148,40 @@ func TestDiscoverOperatorVersion(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      "skyhook-operator-controller-manager",
 					Namespace: "skyhook",
+					Labels: map[string]string{
+						"app.kubernetes.io/name": "skyhook-operator", // needed for detection
+					},
 				},
 			},
 			wantErr: true,
 		},
+		{
+			name: "detection by image name (no skyhook label)",
+			deployment: &appsv1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "custom-named-deployment",
+					Namespace: "skyhook",
+					Labels: map[string]string{
+						"app.kubernetes.io/version": "v1.5.0",
+						"app":                       "my-operator", // no skyhook in labels
+					},
+				},
+				Spec: appsv1.DeploymentSpec{
+					Template: corev1.PodTemplateSpec{
+						Spec: corev1.PodSpec{
+							Containers: []corev1.Container{
+								{
+									Name:  "manager",
+									Image: "nvcr.io/nvidia/skyhook/operator:v1.5.0", // detected by image
+								},
+							},
+						},
+					},
+				},
+			},
+			wantErr: false,
+			wantVer: "v1.5.0",
+		},
 		{
 			name: "latest tag should fail",
 			deployment: &appsv1.Deployment{
@@ -189,7 +221,7 @@ func TestDiscoverOperatorVersion(t *testing.T) {
 				}
 			}
 
-			version, err := discoverOperatorVersion(context.Background(), clientset, "skyhook")
+			version, err := utils.DiscoverOperatorVersion(context.Background(), clientset, "skyhook")
 
 			if (err != nil) != tt.wantErr {
 				t.Errorf("discoverOperatorVersion() error = %v, wantErr %v", err, tt.wantErr)
@@ -244,7 +276,7 @@ func TestExtractImageTag(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got := extractImageTag(tt.image)
+			got := utils.ExtractImageTag(tt.image)
 			if got != tt.want {
 				t.Errorf("extractImageTag(%q) = %q, want %q", tt.image, got, tt.want)
 			}