feat(scheduler,admission): implement in-place pod resize accounting and queue admission (#2021)

Signed-off-by: gshaibi <gshaibi@nvidia.com>

Author: gshaibi

.changes/unreleased/added-20260806-093634.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: |-
+  In-place pod resize accounting (KEP-1287) and queue admission webhook

.changes/unreleased/changed-20260810-114327.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: |-
+  Queue allocated status now includes init-container peak and sidecar requests

cmd/admission/app/app.go

@@ -30,6 +30,8 @@ import (
 
 	kaiv1alpha1 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/kai/v1alpha1"
 	schedulingv1alpha2 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/scheduling/v1alpha2"
+	schedulingv2 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/scheduling/v2"
+	schedulingv2alpha2 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/scheduling/v2alpha2"
 
 	admissionplugins "github.com/kai-scheduler/KAI-scheduler/pkg/admission/plugins"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/admission/webhook/topologyhooks"
@@ -43,6 +45,8 @@ var (
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(schedulingv1alpha2.AddToScheme(scheme))
+	utilruntime.Must(schedulingv2.AddToScheme(scheme))
+	utilruntime.Must(schedulingv2alpha2.AddToScheme(scheme))
 	utilruntime.Must(kaiv1alpha1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
@@ -58,6 +62,7 @@ type App struct {
 
 // +kubebuilder:webhook:path=/mutate--v1-pod,mutating=true,failurePolicy=fail,sideEffects=None,resources=pods,verbs=create,groups=core,versions=v1,name=admission.run.ai,admissionReviewVersions=v1,reinvocationPolicy=IfNeeded
 // +kubebuilder:webhook:path=/validate--v1-pod,mutating=false,failurePolicy=fail,sideEffects=None,resources=pods,verbs=create;update,groups=core,versions=v1,name=admission.run.ai,admissionReviewVersions=v1
+// +kubebuilder:webhook:path=/validate--v1-pod-resize,mutating=false,failurePolicy=ignore,sideEffects=None,resources=pods/resize,verbs=update,groups=core,versions=v1,name=podresize.admission.run.ai,admissionReviewVersions=v1
 // +kubebuilder:webhook:path=/validate-kai-scheduler-v1alpha1-topology,mutating=false,failurePolicy=fail,sideEffects=None,resources=topologies,verbs=create;update,groups=kai.scheduler,versions=v1alpha1,name=topology.admission.run.ai,admissionReviewVersions=v1
 
 func New() (*App, error) {
@@ -134,6 +139,9 @@ func (app *App) RegisterPlugins(admissionPlugins *admissionplugins.KaiAdmissionP
 
 // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch;update
+// +kubebuilder:rbac:groups="scheduling.run.ai",resources=podgroups,verbs=get;list;watch
+// +kubebuilder:rbac:groups="scheduling.run.ai",resources=queues,verbs=get;list;watch
+// +kubebuilder:rbac:groups="scheduling.k8s.io",resources=priorityclasses,verbs=get;list;watch
 
 func (app *App) Run() error {
 	var err error
@@ -150,6 +158,20 @@ func (app *App) Run() error {
 		return err
 	}
 
+	// Add a new webhook for the resize subresource with Ignore failPolicy
+	if err = ctrl.NewWebhookManagedBy(app.manager, &corev1.Pod{}).
+		WithValidator(admissionhooks.NewPodResizeValidator(
+			app.manager.GetClient(),
+			app.Options.SchedulerName,
+			app.Options.ValidatePodResizeQuota,
+			app.Options.BlockUpsizeOnBoundedQueues,
+		)).
+		WithValidatorCustomPath("/validate--v1-pod-resize").
+		Complete(); err != nil {
+		setupLog.Error(err, "unable to create pod resize webhook", "webhook", "PodResize")
+		return err
+	}
+
 	if err = ctrl.NewWebhookManagedBy(app.manager, &kaiv1alpha1.Topology{}).
 		WithValidator(topologyhooks.NewTopologyValidator()).Complete(); err != nil {
 		setupLog.Error(err, "unable to create topology webhook", "webhook", "Topology")

cmd/admission/app/options.go

@@ -26,6 +26,8 @@ type Options struct {
 	BlockNvidiaVisibleDevices   bool
 	GPUPodRuntimeClassName      string
 	GPUFractionRuntimeClassName string
+	ValidatePodResizeQuota      bool
+	BlockUpsizeOnBoundedQueues  bool
 }
 
 // ResolvedGPUFractionRuntimeClassName returns the effective runtime class name
@@ -91,6 +93,15 @@ func InitOptions() *Options {
 		fmt.Sprintf("Runtime class to be set for GPU fraction pods (defaults to %s). "+
 			"Whole-GPU pods are not affected. Set to empty string to disable.",
 			constants.DefaultRuntimeClassName))
+	fs.BoolVar(&options.ValidatePodResizeQuota,
+		"validate-pod-resize-quota", true,
+		"Enable queue limit/quota checks on pod resize requests. "+
+			"Best-effort: if lookups fail, resize is admitted. "+
+			"Ignored if false, disables --block-upsize-on-bounded-queues.")
+	fs.BoolVar(&options.BlockUpsizeOnBoundedQueues,
+		"block-upsize-on-bounded-queues", false,
+		"Block pod upsize if queue or ancestor has a CPU/memory limit. "+
+			"No effect if --validate-pod-resize-quota is false.")
 
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
 

deployments/kai-scheduler/crds/kai.scheduler_configs.yaml

@@ -71,6 +71,27 @@ spec:
                     description: GPUSharing enables GPU sharing functionality for
                       the admission service
                     type: boolean
+                  inPlacePodResize:
+                    description: InPlacePodResize configures in-place pod resize (KEP-1287)
+                      behaviour.
+                    properties:
+                      blockUpsizeOnBoundedQueues:
+                        description: |-
+                          BlockUpsizeOnBoundedQueues rejects any upsize on a queue (or ancestor)
+                          that has a finite CPU or memory limit, regardless of current allocation.
+                          This prevents concurrent-resize races from exceeding hard limits.
+                          Disabled by default; enable only when strict race-free enforcement is
+                          required and best-effort checks are insufficient.
+                          Has no effect when ValidateQuota is false.
+                        type: boolean
+                      validateQuota:
+                        description: |-
+                          ValidateQuota enables best-effort hierarchical queue quota checks on
+                          pods/resize requests. When false, the webhook admits all resizes without
+                          checking queue limits or quota, and BlockUpsizeOnBoundedQueues is ignored.
+                          Defaults to true.
+                        type: boolean
+                    type: object
                   mutatingWebhookConfigurationName:
                     description: MutatingWebhookConfigurationName is the name of the
                       MutatingWebhookConfiguration for the admission service

deployments/kai-scheduler/templates/rbac/admission.yaml

@@ -30,3 +30,20 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - scheduling.k8s.io
+  resources:
+  - priorityclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - scheduling.run.ai
+  resources:
+  - podgroups
+  - queues
+  verbs:
+  - get
+  - list
+  - watch

docs/developer/designs/in-place-pod-resize/README.md

@@ -0,0 +1,161 @@
+<!--
+Copyright 2026 NVIDIA CORPORATION
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# In-Place Pod Resize Accounting and Queue Admission
+
+*Status: Implemented*
+
+Related issues: [#1906](https://github.com/kai-scheduler/KAI-Scheduler/issues/1906),
+[#1872](https://github.com/kai-scheduler/KAI-Scheduler/issues/1872) (deferred resize eviction — separate track)
+
+## Motivation
+
+In-place resize (`pods/resize`) lets a running Pod change CPU/memory without
+rescheduling. KAI has three gaps:
+
+1. **Accounting.** KAI charges Pod resources from the spec. While a resize is
+   pending, infeasible, or in progress, spec ≠ occupancy. That creates phantom
+   capacity on downsize, overcharges on infeasible upsize, and corrupts queue
+   allocation, fair share, reclaim, and reporting.
+2. **Queue admission.** Resize bypasses scheduling. A Pod can grow past queue
+   `limit`, or past deserved `quota` when non-preemptible. The validating
+   webhook matches `pods`, not `pods/resize`, so it never sees an old→new
+   delta.
+3. **Deferred resize eviction.** When the kubelet marks a resize `Deferred`
+   for node capacity, KAI does not preempt or reclaim on that node to make
+   room. Out of scope here — tracked in
+   [#1872](https://github.com/kai-scheduler/KAI-Scheduler/issues/1872).
+
+This design covers (1) and (2). Upstream ResourceQuota already admits resize
+via status-aware requests and a positive usage delta; KAI needs the same
+effective-request model and a hierarchical check.
+
+## Design
+
+### Effective accounting
+
+Use the upstream effective request as KAI's default Pod resource vector. Per
+container and resource, before Pod-level aggregation:
+
+```text
+normal / Deferred / in progress:
+    effective = max(spec request, allocatedResources, status.resources)
+
+Infeasible:
+    effective = max(allocatedResources, status.resources)
+```
+
+Drive node fit, queue `Allocated` / `AllocatedNotPreemptible`, fair share,
+ordering, reclaim, victim selection, and allocated status/metrics from this
+vector. Do not plumb a separate desired-resource vector through the
+scheduler; read the raw spec only for intent (proposed target, user-facing
+desired fields, infeasible diagnostics).
+
+**`Infeasible` detection** delegates to upstream
+`resource.IsPodResizeInfeasible`, which keys off the condition reason alone.
+The kubelet owns the condition lifecycle — it clears or replaces
+`PodResizePending` when a new resize is submitted — so generation tracking is
+not needed, and `observedGeneration` is only populated behind the non-GA
+`PodObservedGenerationTracking` gate (a guard on it would disable `Infeasible`
+handling on gate-off clusters). Upstream semantics are pinned by a
+characterization test so a future upstream tightening surfaces as a failure.
+
+The `Deferred` charge at `max(spec, actual)` is load-bearing beyond
+accounting: deferred-resize eviction
+([#1872](https://github.com/kai-scheduler/KAI-Scheduler/issues/1872),
+[#2051](https://github.com/kai-scheduler/KAI-Scheduler/pull/2051)) relies on
+the deferred target already being reserved in node and queue accounting, so
+capacity freed by evicting victims is not backfilled before the kubelet
+enacts the resize. Charging `Deferred` at actual only would reintroduce
+eviction thrash.
+
+Implementation: upstream `resource.AggregateContainerRequests` (status
+resources enabled), then KAI custom-resource logic.
+
+### Best-effort resize quota admission
+
+Validating webhook on `UPDATE`/`pods/resize` with old and proposed Pods. For
+KAI-scheduled Pods:
+
+1. Resolve PodGroup, preemptibility, leaf queue, and ancestors.
+2. Compute old and proposed effective requests (inherited `Infeasible` stale).
+3. `delta = max(proposed - old, 0)` per resource.
+4. Reject if any queue on the path would violate:
+
+```text
+all workloads:
+    Allocated + delta > limit
+
+non-preemptible:
+    AllocatedNotPreemptible + delta > quota (deserved)
+```
+
+Usage comes from the best-available snapshot (Queue status / live effective
+requests). Downsize always admissible; capacity frees only when effective
+usage falls. Limit-only changes with no request growth have zero delta.
+Rejections name Pod, PodGroup, limiting queue/ancestor, resource, allocation,
+delta, and boundary.
+
+This is **not** atomic across concurrent resizes or against in-flight
+scheduler allocations. Existing violations (race, bypass, pre-rollout) keep
+full effective accounting, are reported, and block further deepenings via the
+webhook and allocate-time capacity checks. No proactive "drain until under
+limit" action for now.
+
+### Optional strict upsize block
+
+Config knob, **disabled by default**, rejects request upsizes that could race
+past a hard queue bound — even if current usage would still fit:
+
+- **Any** upsize, if the leaf queue or an ancestor has a finite `limit` on that
+  resource.
+- **Non-preemptible** upsize, if the leaf queue or an ancestor has a finite
+  `quota` (deserved) on that resource.
+
+Escape hatch for operators who cannot tolerate best-effort races without a
+ledger. Preemptible upsizes on queues that are only bounded by deserved (no
+finite limit) remain subject to the best-effort check only — they may go over
+deserved by design.
+
+```yaml
+spec:
+  admission:
+    inPlacePodResize:
+      validateQuota: true                 # best-effort hierarchical checks
+      blockUpsizeOnBoundedQueues: false   # conservative; default off
+```
+
+Exact CR field names TBD at implementation.
+
+## Decisions
+
+| Topic | Decision |
+| --- | --- |
+| Accounting model | Upstream effective request; upstream reason-only `Infeasible` semantics (kubelet owns the condition lifecycle) |
+| Resize admit path | Keep native `pods/resize`; validate in webhook (do **not** convert to a KAI-owned scheduling API — breaks VPA / API contract) |
+| Non-preemptible growth past quota | Reject at webhook; keep reject at allocate |
+| Concurrent-resize / scheduler races | Best effort; optional `blockUpsizeOnBoundedQueues`; reservation ledger only if a concrete issue appears |
+| Drain-until-under-limit action | Rejected — poor UX; reclaim stays demand-driven |
+| Deferred resize preemption | Separate track: [#1872](https://github.com/kai-scheduler/KAI-Scheduler/issues/1872) / [#2051](https://github.com/kai-scheduler/KAI-Scheduler/pull/2051). Depends on `Deferred` charged at max(spec, actual) — the reserved target is its thrash safety |
+| Wait for upstream resize gates / KEP-5836 only | Rejected as sole plan — ship Goals accounting + admit now; upstream remains complementary ([kubernetes#131835](https://github.com/kubernetes/kubernetes/issues/131835)) |
+
+## Known gaps
+
+Best-effort admission can race: concurrent upsizes, or a resize vs scheduler
+allocate, may double-spend the same remaining headroom.
+`blockUpsizeOnBoundedQueues` and allocate/webhook deepenings mitigate today.
+Prefer upstream resize gates
+([kubernetes#131835](https://github.com/kubernetes/kubernetes/issues/131835))
+as the long-term out-of-tree quota hook; add a reservation ledger only if
+gates do not land and races hurt in production.
+
+## References
+
+- [KEP-1287: In-place Update of Pod Resources](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/1287-in-place-update-pod-resources/README.md)
+- [Kubernetes 1.35 in-place container resize](https://v1-35.docs.kubernetes.io/docs/tasks/configure-pod-container/resize-container-resources/)
+- [Upstream effective Pod request helper](https://github.com/kubernetes/kubernetes/blob/v1.35.4/staging/src/k8s.io/component-helpers/resource/helpers.go)
+- [Upstream Pod quota evaluator](https://github.com/kubernetes/kubernetes/blob/v1.35.4/pkg/quota/v1/evaluator/core/pods.go)
+- [KEP-5836: Scheduler Preemption for In-Place Pod Resize](https://www.kubernetes.dev/resources/keps/5836)
+- [kubernetes#131835: out-of-tree quota vs in-place resize](https://github.com/kubernetes/kubernetes/issues/131835)

docs/in-place-resize/README.md

@@ -0,0 +1,93 @@
+# In-Place Pod Resize
+
+KAI Scheduler supports Kubernetes [in-place pod resize](https://kubernetes.io/docs/tasks/configure-pod-container/resize-container-resources/) (KEP-1287): changing a running pod's CPU and memory requests without restarting it. KAI keeps queue accounting consistent throughout a resize, and applies a best-effort admission check of resize requests against queue limits and quota — a guardrail rather than a strict guarantee (see [Best-effort semantics](#best-effort-semantics)).
+
+Requires Kubernetes 1.33 or newer (the `InPlacePodVerticalScaling` feature gate is enabled by default since 1.33 and GA since 1.35).
+
+## Resizing a pod
+
+Resizes go through the `pods/resize` subresource:
+
+```bash
+kubectl patch pod my-pod --subresource resize \
+  -p '{"spec":{"containers":[{"name":"main","resources":{"requests":{"cpu":"2"}}}]}}'
+```
+
+Only CPU and memory can be resized; GPU resources cannot be changed in place.
+
+## Accounting: what a resizing pod is charged
+
+During a resize, a pod's desired requests (spec) and its actual running size can differ. KAI charges the **effective request** per container and resource:
+
+| Resize state | Charged amount |
+|---|---|
+| No resize in progress | spec |
+| Resize pending or in progress | max(spec, actual) |
+| Resize marked **Infeasible** by the kubelet | actual only — the unreachable spec is not charged |
+
+"Actual" is the value the kubelet reports in the container status. This model matches the upstream Kubernetes scheduler and kubelet, and applies consistently across:
+
+- scheduler decisions (fair share, node fit, preemption and reclaim),
+- queue status (`Queue.status.allocated`, `Queue.status.allocatedNonPreemptible`),
+- resize admission (below).
+
+The practical consequences:
+
+- A pod mid-downsize keeps its current (larger) charge until the kubelet completes the downsize — the freed capacity is never promised before it exists.
+- A pod whose upsize the kubelet rejected as Infeasible does not hold queue capacity for a size it will never reach.
+
+## Resize admission
+
+A validating webhook on `pods/resize` checks every upsize of a KAI-scheduled pod against the pod's queue hierarchy, before the resize is persisted:
+
+- **Limit** (all workloads): the increase must not push any queue's `allocated` over its CPU or memory `limit`.
+- **Quota** (non-preemptible workloads only): the increase must also fit within `quota`, checked against `allocatedNonPreemptible`.
+
+The increase is computed at the pod level, so moving CPU or memory between containers of the same pod is not counted as growth, and downsizes are always admitted.
+
+A rejected resize returns an error such as:
+
+```
+resize rejected: pod team-a/trainer (PodGroup pg-trainer) CPU upsize would push
+queue team-a Allocated (1000m + 2000m) over limit (2000m)
+```
+
+The pod keeps running at its current size; nothing is restarted or evicted.
+
+### Best-effort semantics
+
+Admission is a guardrail, not a transactional guarantee:
+
+- The webhook is registered with `failurePolicy: Ignore` — if the admission service is unavailable, resizes are admitted rather than blocked.
+- Lookup failures (queue, podgroup) admit the resize and log an error.
+- The check reads `Queue.status.allocated`, which trails actual scheduling decisions through the controller reconcile chain. A resize can therefore race concurrent activity on a nearly-full queue — another resize, or the scheduler placing new pods — and together exceed the limit. The reverse also holds: a recently freed queue may transiently over-deny a valid resize. The scheduler's own accounting stays correct either way and stops further allocation once over the limit.
+- A resize issued while the same pod's previous resize is still being enacted is checked against the transient charge. In particular, a pending downsize is still charged until the kubelet enacts it, so a follow-up upsize within that window can reclaim the not-yet-released capacity and settle above the limit.
+
+For workloads where the race above is unacceptable, see `blockUpsizeOnBoundedQueues` below.
+
+## Configuration
+
+Resize admission is configured in the KAI `Config` resource:
+
+```yaml
+apiVersion: kai.scheduler/v1
+kind: Config
+metadata:
+  name: kai
+spec:
+  admission:
+    inPlacePodResize:
+      validateQuota: true              # default
+      blockUpsizeOnBoundedQueues: false # default
+```
+
+| Field | Default | Effect |
+|---|---|---|
+| `validateQuota` | `true` | Master switch. When `false`, the webhook admits all resizes without any checks, and `blockUpsizeOnBoundedQueues` is ignored. |
+| `blockUpsizeOnBoundedQueues` | `false` | Strict mode: reject **any** upsize on a queue (or ancestor) that has a finite CPU or memory limit — regardless of free headroom — and, for non-preemptible workloads, on any queue with a finite quota. Closes the concurrent-resize race at the cost of disallowing upsizes on bounded queues entirely. |
+
+## Interaction with queue limits
+
+`limit: -1` on a queue resource means unbounded; any other value (including `0`) is a hard bound enforced on resize. The same convention applies to `quota` for non-preemptible workloads. See [Queues](../queues/README.md) for how limits and quota are configured.
+
+Because checks walk the full queue hierarchy, an upsize must fit under every ancestor's limit, not just the leaf queue's.