feat: add sysreq based manual reboot

Signed-off-by: Ajay Mishra <ajmishra@nvidia.com>

Author: XRFXLP

distros/kubernetes/nvsentinel/charts/janitor-provider/templates/deployment.yaml

@@ -82,6 +82,8 @@ spec:
             # Generic provider environment variables
             - name: GENERIC_REBOOT_IMAGE
               value: {{ .Values.csp.generic.rebootImage | quote }}
+            - name: GENERIC_REBOOT_USE_SYSRQ
+              value: {{ .Values.csp.generic.useSysrqReboot | default false | quote }}
             - name: GENERIC_REBOOT_JOB_NAMESPACE
               value: {{ .Values.csp.generic.rebootJobNamespace | default .Release.Namespace | quote }}
             - name: GENERIC_REBOOT_JOB_TTL

distros/kubernetes/nvsentinel/charts/janitor-provider/values.yaml

@@ -106,13 +106,16 @@ csp:
   # - azure: For Microsoft Azure AKS clusters
   # - oci: For Oracle Cloud Infrastructure OKE clusters
   # - nebius: For Nebius Managed Kubernetes (MK8s) clusters
-  # - generic: For bare-metal / on-premises clusters (reboots via privileged Job running chroot /host reboot)
+  # - generic: For bare-metal / on-premises clusters (reboots via privileged Job)
   provider: "kind"
 
   # Generic provider configuration (only used when provider=generic)
   generic:
     # Container image used for the reboot Job
     rebootImage: "public.ecr.aws/docker/library/busybox:1.37.0"
+    # Use the Linux Magic SysRq trigger for an immediate kernel reboot instead of
+    # running chroot /host reboot.
+    useSysrqReboot: false
     # Namespace for reboot Jobs (defaults to the janitor-provider's namespace)
     rebootJobNamespace: ""
     # TTL in seconds for completed reboot Jobs (auto-cleanup)

docs/configuration/README.md

@@ -104,5 +104,6 @@ Each module has additional configuration options documented in its dedicated gui
 - [Fault Quarantine](./fault-quarantine.md)
 - [Node Drainer](./node-drainer.md)
 - [Fault Remediation](./fault-remediation.md)
+- [Janitor Provider](./janitor-provider.md)
 - [Preflight](./preflight.md)
 - [Event Exporter](./event-exporter.md)

docs/configuration/janitor-provider.md

@@ -0,0 +1,63 @@
+# Janitor Provider Configuration
+
+## Overview
+
+The janitor-provider module executes node lifecycle operations requested by Janitor, including reboot signals, node readiness checks, and provider-specific termination signals. This document covers the Helm configuration options for selecting and configuring the provider backend.
+
+## Configuration Reference
+
+### Module Enable/Disable
+
+Controls whether the janitor-provider module is deployed in the cluster.
+
+```yaml
+global:
+  janitorProvider:
+    enabled: true
+```
+
+### CSP Provider
+
+Selects the provider implementation used by janitor-provider.
+
+```yaml
+janitor-provider:
+  csp:
+    provider: "kind"
+```
+
+Supported providers are `kind`, `kwok`, `aws`, `gcp`, `azure`, `oci`, `nebius`, and `generic`.
+
+## Generic Provider
+
+The `generic` provider is intended for bare-metal and on-premises clusters where there is no cloud provider reboot API. It creates a privileged Kubernetes Job on the target node and uses the node `bootID` to verify that a reboot occurred.
+
+```yaml
+janitor-provider:
+  csp:
+    provider: "generic"
+    generic:
+      rebootImage: "public.ecr.aws/docker/library/busybox:1.37.0"
+      useSysrqReboot: false
+      rebootJobNamespace: ""
+      rebootJobTTLSeconds: 3600
+      imagePullSecrets: ""
+```
+
+### Generic Provider Options
+
+`rebootImage` sets the container image used for the privileged reboot Job.
+
+`useSysrqReboot` switches the reboot command from `chroot /host reboot` to Linux Magic SysRq by writing `b` to the host `/proc/sysrq-trigger`. Keep this disabled unless the standard reboot path leaves nodes stuck `NotReady`.
+
+`rebootJobNamespace` sets the namespace where reboot Jobs are created. If empty, the janitor-provider release namespace is used.
+
+`rebootJobTTLSeconds` controls how long completed reboot Jobs are retained before Kubernetes garbage-collects them.
+
+`imagePullSecrets` is a comma-separated list of image pull secret names used by the reboot Job.
+
+### SysRq Reboot
+
+When `useSysrqReboot` is enabled, janitor-provider sets `GENERIC_REBOOT_USE_SYSRQ=true` and the reboot Job writes to the host SysRq trigger instead of invoking the normal reboot command.
+
+This path bypasses the normal userspace shutdown flow. It is useful for environments where `chroot /host reboot` or `sudo reboot` is accepted but leaves the node stuck `NotReady`, but it should remain an explicit opt-in because it is more abrupt than the default reboot path.

docs/designs/028-generic-baremetal-reboot-provider.md

@@ -33,7 +33,7 @@ flowchart TD
 
 ## Decision
 
-Add a **generic provider** (`CSP=generic`) that reboots nodes via a privileged Kubernetes Job running `chroot /host /sbin/reboot`, following the Job-based pattern from GPU Reset ([ADR-019](019-janitor-gpu-reset.md)). It is a named provider in the factory switch, just like `aws`, `gcp`, or `kind`.
+Add a **generic provider** (`CSP=generic`) that reboots nodes via a privileged Kubernetes Job. By default the Job runs `chroot /host /sbin/reboot`; deployments can opt into a Linux Magic SysRq reboot for environments where the normal reboot path wedges the node. This follows the Job-based pattern from GPU Reset ([ADR-019](019-janitor-gpu-reset.md)). It is a named provider in the factory switch, just like `aws`, `gcp`, or `kind`.
 
 ## Implementation
 
@@ -69,7 +69,7 @@ sequenceDiagram
     GP-->>JC: requestID = pre-reboot bootID
 
     K8s->>Job: Schedule pod on target node
-    Job->>Node: chroot /host /sbin/reboot
+    Job->>Node: chroot /host /sbin/reboot or echo b > /proc/sysrq-trigger
     Note over Node: Node reboots, pod is killed
 
     Note over Node: Node boots back up, new bootID assigned
@@ -87,6 +87,13 @@ sequenceDiagram
 
 Records the node's current `bootID` from `node.Status.NodeInfo.BootID`, creates a privileged Job on the target node, and returns the pre-reboot `bootID` as the `requestID`.
 
+The reboot Job supports two reboot paths:
+
+- Default: `chroot /host reboot`
+- SysRq opt-in: `echo b > /proc/sysrq-trigger` via the host `/proc` mount
+
+The SysRq path is intended for bare-metal environments where the standard reboot command is accepted but leaves the node stuck `NotReady`. It bypasses the normal userspace shutdown path, so it is controlled by an explicit feature flag.
+
 **Job specification:**
 
 ```yaml
@@ -166,6 +173,7 @@ csp:
 
   generic:                      # config for the generic provider (when provider=generic)
     rebootImage: "busybox:1.37"
+    useSysrqReboot: false       # true to use echo b > /proc/sysrq-trigger
     rebootJobNamespace: ""      # defaults to the janitor-provider's own namespace
     rebootJobTTLSeconds: 3600
 ```
@@ -177,6 +185,8 @@ env:
     value: {{ .Values.csp.provider | default "kind" | quote }}
   - name: GENERIC_REBOOT_IMAGE
     value: {{ .Values.csp.generic.rebootImage | default "busybox:1.37" | quote }}
+  - name: GENERIC_REBOOT_USE_SYSRQ
+    value: {{ .Values.csp.generic.useSysrqReboot | default false | quote }}
   - name: GENERIC_REBOOT_JOB_NAMESPACE
     value: {{ .Values.csp.generic.rebootJobNamespace | quote }}
   - name: GENERIC_REBOOT_JOB_TTL

janitor-provider/pkg/csp/generic/generic.go

@@ -38,20 +38,22 @@ const (
 	jobLabelKey                = "nvsentinel.nvidia.com/reboot-job"
 	jobNodeLabelKey            = "nvsentinel.nvidia.com/reboot-node"
 	hostMountPath              = "/host"
+	hostProcMountPath          = "/host-proc"
 )
 
 var _ model.CSPClient = (*Client)(nil)
 
 // Config holds the configuration for the generic provider.
 type Config struct {
 	RebootImage          string
+	UseSysrqReboot       bool
 	RebootJobNamespace   string
 	RebootJobTTL         int32
 	RebootJobPullSecrets []string
 }
 
 // Client is the generic bare-metal implementation of the CSP Client interface.
-// It reboots nodes by creating a privileged Job that runs chroot /host reboot.
+// It reboots nodes by creating a privileged Job that runs the configured reboot method.
 type Client struct {
 	k8sClient kubernetes.Interface
 	config    Config
@@ -89,7 +91,7 @@ func NewClientWithK8s(k8sClient kubernetes.Interface, config Config) *Client {
 }
 
 // SendRebootSignal creates a privileged Job on the target node that executes
-// chroot /host reboot. Returns the node's pre-reboot bootID as the requestID.
+// the configured reboot command. Returns the node's pre-reboot bootID as the requestID.
 func (c *Client) SendRebootSignal(ctx context.Context, node corev1.Node) (model.ResetSignalRequestRef, error) {
 	preRebootBootID := node.Status.NodeInfo.BootID
 	if preRebootBootID == "" {
@@ -99,15 +101,17 @@ func (c *Client) SendRebootSignal(ctx context.Context, node corev1.Node) (model.
 
 	job := c.buildRebootJob(node.Name)
 
-	slog.InfoContext(ctx, "Creating reboot Job", "node", node.Name, "namespace", c.config.RebootJobNamespace)
+	slog.InfoContext(ctx, "Creating reboot Job", "node", node.Name, "namespace", c.config.RebootJobNamespace,
+		"useSysrqReboot", c.config.UseSysrqReboot)
 
 	created, err := c.k8sClient.BatchV1().Jobs(c.config.RebootJobNamespace).Create(ctx, job, metav1.CreateOptions{})
 	if err != nil {
 		return "", fmt.Errorf("failed to create reboot job for node %s: %w", node.Name, err)
 	}
 
 	slog.InfoContext(ctx, "Reboot Job created", "node", node.Name, "job", created.Name,
-		"jobNamespace", c.config.RebootJobNamespace, "bootID", preRebootBootID)
+		"jobNamespace", c.config.RebootJobNamespace, "bootID", preRebootBootID,
+		"useSysrqReboot", c.config.UseSysrqReboot)
 
 	return model.ResetSignalRequestRef(preRebootBootID), nil
 }
@@ -151,6 +155,37 @@ func (c *Client) SendTerminateSignal(ctx context.Context, node corev1.Node) (mod
 func (c *Client) buildRebootJob(nodeName string) *batchv1.Job {
 	image := c.config.RebootImage
 	ttl := c.config.RebootJobTTL
+	command := c.config.rebootCommand()
+	volumeMounts := []corev1.VolumeMount{
+		{
+			Name:      "host-root",
+			MountPath: hostMountPath,
+		},
+	}
+	volumes := []corev1.Volume{
+		{
+			Name: "host-root",
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: "/",
+				},
+			},
+		},
+	}
+	if c.config.UseSysrqReboot {
+		volumeMounts = append(volumeMounts, corev1.VolumeMount{
+			Name:      "host-proc",
+			MountPath: hostProcMountPath,
+		})
+		volumes = append(volumes, corev1.Volume{
+			Name: "host-proc",
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: "/proc",
+				},
+			},
+		})
+	}
 
 	return &batchv1.Job{
 		ObjectMeta: metav1.ObjectMeta{
@@ -182,34 +217,28 @@ func (c *Client) buildRebootJob(nodeName string) *batchv1.Job {
 						{
 							Name:    "reboot",
 							Image:   image,
-							Command: []string{"chroot", hostMountPath, "reboot"},
+							Command: command,
 							SecurityContext: &corev1.SecurityContext{
 								Privileged: ptr.To(true),
 							},
-							VolumeMounts: []corev1.VolumeMount{
-								{
-									Name:      "host-root",
-									MountPath: hostMountPath,
-								},
-							},
-						},
-					},
-					Volumes: []corev1.Volume{
-						{
-							Name: "host-root",
-							VolumeSource: corev1.VolumeSource{
-								HostPath: &corev1.HostPathVolumeSource{
-									Path: "/",
-								},
-							},
+							VolumeMounts: volumeMounts,
 						},
 					},
+					Volumes: volumes,
 				},
 			},
 		},
 	}
 }
 
+func (c Config) rebootCommand() []string {
+	if c.UseSysrqReboot {
+		return []string{"sh", "-c", fmt.Sprintf("echo b > %s/sysrq-trigger", hostProcMountPath)}
+	}
+
+	return []string{"chroot", hostMountPath, "reboot"}
+}
+
 // checkRebootJobPodStatus checks the reboot Job's pod for terminal failures
 // (e.g., ImagePullBackOff) that indicate the reboot was never attempted.
 func (c *Client) checkRebootJobPodStatus(ctx context.Context, nodeName string) error {
@@ -319,6 +348,16 @@ func loadConfigFromEnv() Config {
 		image = defaultRebootImage
 	}
 
+	useSysrqReboot := false
+	if useSysrqStr := os.Getenv("GENERIC_REBOOT_USE_SYSRQ"); useSysrqStr != "" {
+		parsed, err := strconv.ParseBool(useSysrqStr)
+		if err != nil {
+			slog.Warn("Invalid GENERIC_REBOOT_USE_SYSRQ, using default", "value", useSysrqStr, "default", false)
+		} else {
+			useSysrqReboot = parsed
+		}
+	}
+
 	namespace := os.Getenv("GENERIC_REBOOT_JOB_NAMESPACE")
 	ttl := int32(defaultRebootJobTTLSeconds)
 
@@ -343,6 +382,7 @@ func loadConfigFromEnv() Config {
 
 	return Config{
 		RebootImage:          image,
+		UseSysrqReboot:       useSysrqReboot,
 		RebootJobNamespace:   namespace,
 		RebootJobTTL:         ttl,
 		RebootJobPullSecrets: pullSecrets,