fix: deadlock if reboot pods are missing, adds them back

also increase the resource request and limits.
we at times were seeing some pods hanging in different clusters

Author: lockwobr

chart/templates/skyhook-crd.yaml

@@ -390,14 +390,14 @@ spec:
                           anyOf:
                           - type: integer
                           - type: string
-                          default: 250m
+                          default: 500m
                           pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                           x-kubernetes-int-or-string: true
                         cpuRequest:
                           anyOf:
                           - type: integer
                           - type: string
-                          default: 50m
+                          default: 500m
                           pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                           x-kubernetes-int-or-string: true
                         memoryLimit:
@@ -411,7 +411,7 @@ spec:
                           anyOf:
                           - type: integer
                           - type: string
-                          default: 64Mi
+                          default: 256Mi
                           pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                           x-kubernetes-int-or-string: true
                       type: object

k8s-tests/chainsaw/skyhook/interrupt-grouping/skyhook.yaml

@@ -34,4 +34,4 @@ spec:
         type: service
         services: [containerd, foobar]
       resources:
-        cpuLimit: 123m
+        cpuLimit: 523m

k8s-tests/chainsaw/skyhook/uninstall-upgrade-skyhook/assert-update.yaml

@@ -30,23 +30,23 @@ spec:
         (length(@)): 3
       resources: ## including these to tests defaults are working
         limits:
-          cpu: 250m
+          cpu: 500m
           memory: 256Mi
         requests:
-          cpu: 50m
-          memory: 64Mi
+          cpu: 500m
+          memory: 256Mi
     - name: cats-uninstallcheck
       args:
         ([0]): uninstall-check
         ([1]): /root
         (length(@)): 3
       resources: ## including these to tests defaults are working
         limits:
-          cpu: 250m
+          cpu: 500m
           memory: 256Mi
         requests:
-          cpu: 50m
-          memory: 64Mi
+          cpu: 500m
+          memory: 256Mi
 ---
 kind: Pod
 apiVersion: v1

k8s-tests/chainsaw/skyhook/validate-packages/update.yaml

@@ -19,9 +19,9 @@ spec:
       version: "1.2.3"
       image: ghcr.io/nvidia/skyhook/agentless
       resources:
-        cpuRequest: 50m
-        cpuLimit: 250m
-        memoryRequest: 64Mi
+        cpuRequest: 500m
+        cpuLimit: 500m
+        memoryRequest: 256Mi
         memoryLimit: 256Mi
     invalid-env:
       version: "5.4.3"

operator/api/v1alpha1/skyhook_types.go

@@ -171,11 +171,11 @@ func (p *PackageRef) GetUniqueName() string {
 }
 
 type ResourceRequirements struct {
-	// +kubebuilder:default="50m"
+	// +kubebuilder:default="500m"
 	CPURequest resource.Quantity `json:"cpuRequest,omitempty"`
-	// +kubebuilder:default="250m"
+	// +kubebuilder:default="500m"
 	CPULimit resource.Quantity `json:"cpuLimit,omitempty"`
-	// +kubebuilder:default="64Mi"
+	// +kubebuilder:default="256Mi"
 	MemoryRequest resource.Quantity `json:"memoryRequest,omitempty"`
 	// +kubebuilder:default="256Mi"
 	MemoryLimit resource.Quantity `json:"memoryLimit,omitempty"`

operator/config/crd/bases/skyhook.nvidia.com_skyhooks.yaml

@@ -377,14 +377,14 @@ spec:
                           anyOf:
                           - type: integer
                           - type: string
-                          default: 250m
+                          default: 500m
                           pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                           x-kubernetes-int-or-string: true
                         cpuRequest:
                           anyOf:
                           - type: integer
                           - type: string
-                          default: 50m
+                          default: 500m
                           pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                           x-kubernetes-int-or-string: true
                         memoryLimit:
@@ -398,7 +398,7 @@ spec:
                           anyOf:
                           - type: integer
                           - type: string
-                          default: 64Mi
+                          default: 256Mi
                           pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
                           x-kubernetes-int-or-string: true
                       type: object

operator/internal/controller/skyhook_controller.go

@@ -1467,12 +1467,12 @@ func (r *SkyhookReconciler) CreateInterruptPodForPackage(_interrupt *v1alpha1.In
 					VolumeMounts: volumeMounts,
 					Resources: corev1.ResourceRequirements{
 						Limits: corev1.ResourceList{
-							corev1.ResourceCPU:    resource.MustParse("100m"),
-							corev1.ResourceMemory: resource.MustParse("20Mi"),
+							corev1.ResourceCPU:    resource.MustParse("500m"),
+							corev1.ResourceMemory: resource.MustParse("256Mi"),
 						},
 						Requests: corev1.ResourceList{
-							corev1.ResourceCPU:    resource.MustParse("100m"),
-							corev1.ResourceMemory: resource.MustParse("20Mi"),
+							corev1.ResourceCPU:    resource.MustParse("500m"),
+							corev1.ResourceMemory: resource.MustParse("256Mi"),
 						},
 					},
 				},
@@ -1633,14 +1633,14 @@ func (r *SkyhookReconciler) CreatePodFromPackage(_package *v1alpha1.Package, sky
 						Privileged: ptr(true),
 					},
 					VolumeMounts: volumeMounts,
-					Resources: corev1.ResourceRequirements{ // setting this
+					Resources: corev1.ResourceRequirements{
 						Limits: corev1.ResourceList{
-							corev1.ResourceCPU:    resource.MustParse("100m"),
-							corev1.ResourceMemory: resource.MustParse("20Mi"),
+							corev1.ResourceCPU:    resource.MustParse("500m"),
+							corev1.ResourceMemory: resource.MustParse("256Mi"),
 						},
 						Requests: corev1.ResourceList{
-							corev1.ResourceCPU:    resource.MustParse("100m"),
-							corev1.ResourceMemory: resource.MustParse("20Mi"),
+							corev1.ResourceCPU:    resource.MustParse("500m"),
+							corev1.ResourceMemory: resource.MustParse("256Mi"),
 						},
 					},
 				},
@@ -1896,6 +1896,17 @@ func (r *SkyhookReconciler) ProcessInterrupt(ctx context.Context, skyhookNode wr
 		return false, nil
 	}
 
+	// Theres is a race condition when a node reboots and api might clean up the interrupt pod
+	// so we need to check if the pod exists and if it does, we need to recreate it
+	if status != nil && (status.State == v1alpha1.StateInProgress || status.State == v1alpha1.StateErroring) && status.Stage == v1alpha1.StageInterrupt {
+		// call interrupt to recreate the pod if missing
+		// this is safe because the ageent is idempotent
+		err = r.Interrupt(ctx, skyhookNode, _package, interrupt)
+		if err != nil {
+			return false, err
+		}
+	}
+
 	if nextStage != nil && *nextStage == v1alpha1.StageInterrupt && runInterrupt { // time to do the interrupt
 
 		hasWork, err := r.HasNonInterruptWork(ctx, skyhookNode)