feat(scheduler,admission): implement in-place pod resize accounting and queue admission

.changes/unreleased/added-20260806-093634.yaml

@@ -0,0 +1,3 @@
+kind: Added
+body: |-
+  In-place pod resize accounting (KEP-1287) and queue admission webhook

.changes/unreleased/changed-20260810-114327.yaml

@@ -0,0 +1,3 @@
+kind: Changed
+body: |-
+  Queue allocated status now includes init-container peak and sidecar requests

README.md

@@ -43,6 +43,7 @@ It can run alongside other schedulers installed on the cluster.
 - [Min-guaranteed-runtime](https://github.com/kai-scheduler/KAI-scheduler/tree/main/docs/developer/designs/min-runtime): ensures a time period in which the scheduler must not preempt or reclaim a running workload, even if preemptible.
 - Workload Consolidation: Reallocate running workloads intelligently to reduce fragmentation and increase cluster utilization.
 - [Elastic Workloads](docs/elastic/README.md): Dynamically scale workloads within defined minimum and maximum pod or SubGroup thresholds.
+- [In-Place Pod Resize](docs/in-place-resize/README.md): Accurate queue accounting for in-place CPU/memory resizes (KEP-1287), with admission checks against queue limits and quota.
 - Dynamic Resource Allocation (DRA): Support vendor-specific hardware resources through Kubernetes ResourceClaims (e.g., GPUs from NVIDIA or AMD).
 - [Topology-Aware Scheduling (TAS)](https://github.com/kai-scheduler/KAI-scheduler/tree/main/docs/topology): supports optimized placement with [topology aware scheduling](https://github.com/kai-scheduler/KAI-scheduler/tree/main/docs/developer/designs/topology-awareness) and hierarchical topology aware scheduling for [Hierarchical PodGroups](https://github.com/kai-scheduler/KAI-scheduler/tree/main/docs/developer/designs/hierarchical-podgroup).
 - [Hierarchical PodGroups](https://github.com/kai-scheduler/KAI-scheduler/tree/main/docs/developer/designs/hierarchical-podgroup): supports gang scheduling with optimized topology aware scheduling of multi-level workloads, such as distributed and disaggregated workloads such as Dynamo/Grove.

cmd/admission/app/app.go

@@ -30,6 +30,8 @@ import (
 
 	kaiv1alpha1 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/kai/v1alpha1"
 	schedulingv1alpha2 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/scheduling/v1alpha2"
+	schedulingv2 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/scheduling/v2"
+	schedulingv2alpha2 "github.com/kai-scheduler/KAI-scheduler/pkg/apis/scheduling/v2alpha2"
 
 	admissionplugins "github.com/kai-scheduler/KAI-scheduler/pkg/admission/plugins"
 	"github.com/kai-scheduler/KAI-scheduler/pkg/admission/webhook/topologyhooks"
@@ -44,6 +46,8 @@ var (
 func init() {
 	utilruntime.Must(clientgoscheme.AddToScheme(scheme))
 	utilruntime.Must(schedulingv1alpha2.AddToScheme(scheme))
+	utilruntime.Must(schedulingv2.AddToScheme(scheme))
+	utilruntime.Must(schedulingv2alpha2.AddToScheme(scheme))
 	utilruntime.Must(kaiv1alpha1.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }
@@ -60,6 +64,7 @@ type App struct {
 
 // +kubebuilder:webhook:path=/mutate--v1-pod,mutating=true,failurePolicy=fail,sideEffects=None,resources=pods,verbs=create,groups=core,versions=v1,name=admission.run.ai,admissionReviewVersions=v1,reinvocationPolicy=IfNeeded
 // +kubebuilder:webhook:path=/validate--v1-pod,mutating=false,failurePolicy=fail,sideEffects=None,resources=pods,verbs=create;update,groups=core,versions=v1,name=admission.run.ai,admissionReviewVersions=v1
+// +kubebuilder:webhook:path=/validate--v1-pod-resize,mutating=false,failurePolicy=ignore,sideEffects=None,resources=pods/resize,verbs=update,groups=core,versions=v1,name=podresize.admission.run.ai,admissionReviewVersions=v1
 // +kubebuilder:webhook:path=/validate-kai-scheduler-v1alpha1-topology,mutating=false,failurePolicy=fail,sideEffects=None,resources=topologies,verbs=create;update,groups=kai.scheduler,versions=v1alpha1,name=topology.admission.run.ai,admissionReviewVersions=v1
 
 func New() (*App, error) {
@@ -143,6 +148,9 @@ func (app *App) RegisterPlugins(admissionPlugins *admissionplugins.KaiAdmissionP
 // +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=coordination.k8s.io,resources=leases,verbs=get;list;watch;create;update;patch;delete
 // +kubebuilder:rbac:groups=core,resources=events,verbs=create;patch;update
+// +kubebuilder:rbac:groups="scheduling.run.ai",resources=podgroups,verbs=get;list;watch
+// +kubebuilder:rbac:groups="scheduling.run.ai",resources=queues,verbs=get;list;watch
+// +kubebuilder:rbac:groups="scheduling.k8s.io",resources=priorityclasses,verbs=get;list;watch
 
 func (app *App) Run() error {
 	var err error
@@ -159,6 +167,21 @@ func (app *App) Run() error {
 		return err
 	}
 
+	// The canonical /validate--v1-pod path is taken by the pod validator above,
+	// so the resize endpoint registers under an explicit custom path.
+	if err = ctrl.NewWebhookManagedBy(app.manager, &corev1.Pod{}).
+		WithValidator(admissionhooks.NewPodResizeValidator(
+			app.manager.GetClient(),
+			app.Options.SchedulerName,
+			app.Options.ValidatePodResizeQuota,
+			app.Options.BlockUpsizeOnBoundedQueues,
+		)).
+		WithValidatorCustomPath("/validate--v1-pod-resize").
+		Complete(); err != nil {
+		setupLog.Error(err, "unable to create pod resize webhook", "webhook", "PodResize")
+		return err
+	}
+
 	if err = ctrl.NewWebhookManagedBy(app.manager, &kaiv1alpha1.Topology{}).
 		WithValidator(topologyhooks.NewTopologyValidator()).Complete(); err != nil {
 		setupLog.Error(err, "unable to create topology webhook", "webhook", "Topology")

cmd/admission/app/options.go

@@ -28,6 +28,8 @@ type Options struct {
 	BlockNvidiaVisibleDevices   bool
 	GPUPodRuntimeClassName      string
 	GPUFractionRuntimeClassName string
+	ValidatePodResizeQuota      bool
+	BlockUpsizeOnBoundedQueues  bool
 }
 
 // ResolvedGPUFractionRuntimeClassName returns the effective runtime class name
@@ -99,6 +101,16 @@ func InitOptions() *Options {
 		fmt.Sprintf("Runtime class to be set for GPU fraction pods (defaults to %s). "+
 			"Whole-GPU pods are not affected. Set to empty string to disable.",
 			constants.DefaultRuntimeClassName))
+	fs.BoolVar(&options.ValidatePodResizeQuota,
+		"validate-pod-resize-quota", true,
+		"Enable best-effort hierarchical queue quota checks on pods/resize requests. "+
+			"When false, the webhook admits all resizes without checking queue limits or quota, "+
+			"and --block-upsize-on-bounded-queues is ignored.")
+	fs.BoolVar(&options.BlockUpsizeOnBoundedQueues,
+		"block-upsize-on-bounded-queues", false,
+		"Reject any upsize on a queue (or ancestor) with a finite CPU or memory limit, "+
+			"regardless of current allocation. Prevents concurrent-resize races from exceeding hard limits. "+
+			"Has no effect when --validate-pod-resize-quota is false.")
 
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
 

deployments/kai-scheduler/crds/kai.scheduler_configs.yaml

@@ -71,6 +71,27 @@ spec:
                     description: GPUSharing enables GPU sharing functionality for
                       the admission service
                     type: boolean
+                  inPlacePodResize:
+                    description: InPlacePodResize configures in-place pod resize (KEP-1287)
+                      behaviour.
+                    properties:
+                      blockUpsizeOnBoundedQueues:
+                        description: |-
+                          BlockUpsizeOnBoundedQueues rejects any upsize on a queue (or ancestor)
+                          that has a finite CPU or memory limit, regardless of current allocation.
+                          This prevents concurrent-resize races from exceeding hard limits.
+                          Disabled by default; enable only when strict race-free enforcement is
+                          required and best-effort checks are insufficient.
+                          Has no effect when ValidateQuota is false.
+                        type: boolean
+                      validateQuota:
+                        description: |-
+                          ValidateQuota enables best-effort hierarchical queue quota checks on
+                          pods/resize requests. When false, the webhook admits all resizes without
+                          checking queue limits or quota, and BlockUpsizeOnBoundedQueues is ignored.
+                          Defaults to true.
+                        type: boolean
+                    type: object
                   mutatingWebhookConfigurationName:
                     description: MutatingWebhookConfigurationName is the name of the
                       MutatingWebhookConfiguration for the admission service

deployments/kai-scheduler/templates/rbac/admission.yaml

@@ -42,3 +42,20 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - scheduling.k8s.io
+  resources:
+  - priorityclasses
+  verbs:
+  - get
+  - list
+  - watch
+- apiGroups:
+  - scheduling.run.ai
+  resources:
+  - podgroups
+  - queues
+  verbs:
+  - get
+  - list
+  - watch

docs/developer/designs/in-place-pod-resize/README.md

@@ -0,0 +1,161 @@
+<!--
+Copyright 2026 NVIDIA CORPORATION
+SPDX-License-Identifier: Apache-2.0
+-->
+
+# In-Place Pod Resize Accounting and Queue Admission
+
+*Status: Proposed*
+
+Related issues: [#1906](https://github.com/kai-scheduler/KAI-Scheduler/issues/1906),
+[#1872](https://github.com/kai-scheduler/KAI-Scheduler/issues/1872) (deferred resize eviction — separate track)
+
+## Motivation
+
+In-place resize (`pods/resize`) lets a running Pod change CPU/memory without
+rescheduling. KAI has three gaps:
+
+1. **Accounting.** KAI charges Pod resources from the spec. While a resize is
+   pending, infeasible, or in progress, spec ≠ occupancy. That creates phantom
+   capacity on downsize, overcharges on infeasible upsize, and corrupts queue
+   allocation, fair share, reclaim, and reporting.
+2. **Queue admission.** Resize bypasses scheduling. A Pod can grow past queue
+   `limit`, or past deserved `quota` when non-preemptible. The validating
+   webhook matches `pods`, not `pods/resize`, so it never sees an old→new
+   delta.
+3. **Deferred resize eviction.** When the kubelet marks a resize `Deferred`
+   for node capacity, KAI does not preempt or reclaim on that node to make
+   room. Out of scope here — tracked in
+   [#1872](https://github.com/kai-scheduler/KAI-Scheduler/issues/1872).
+
+This design covers (1) and (2). Upstream ResourceQuota already admits resize
+via status-aware requests and a positive usage delta; KAI needs the same
+effective-request model and a hierarchical check.
+
+## Design
+
+### Effective accounting
+
+Use the upstream effective request (same pattern as Volcano) as KAI's default
+Pod resource vector. Per container and resource, before Pod-level aggregation:
+
+```text
+normal / Deferred / in progress:
+    effective = max(spec request, allocatedResources, status.resources)
+
+Infeasible:
+    effective = max(allocatedResources, status.resources)
+```
+
+Drive node fit, queue `Allocated` / `AllocatedNotPreemptible`, fair share,
+ordering, reclaim, victim selection, and allocated status/metrics from this
+vector. Do not plumb a separate desired-resource vector through the
+scheduler; read the raw spec only for intent (proposed target, user-facing
+desired fields, infeasible diagnostics).
+
+**`Infeasible` detection** delegates to upstream
+`resource.IsPodResizeInfeasible`, which keys off the condition reason alone.
+The kubelet owns the condition lifecycle — it clears or replaces
+`PodResizePending` when a new resize is submitted — so generation tracking is
+not needed, and `observedGeneration` is only populated behind the non-GA
+`PodObservedGenerationTracking` gate (a guard on it would disable `Infeasible`
+handling on gate-off clusters). Upstream semantics are pinned by a
+characterization test so a future upstream tightening surfaces as a failure.
+
+The `Deferred` charge at `max(spec, actual)` is load-bearing beyond
+accounting: deferred-resize eviction
+([#1872](https://github.com/kai-scheduler/KAI-Scheduler/issues/1872),
+[#2051](https://github.com/kai-scheduler/KAI-Scheduler/pull/2051)) relies on
+the deferred target already being reserved in node and queue accounting, so
+capacity freed by evicting victims is not backfilled before the kubelet
+enacts the resize. Charging `Deferred` at actual only would reintroduce
+eviction thrash.
+
+Implementation: upstream `resource.AggregateContainerRequests` (status
+resources enabled), then KAI custom-resource logic.
+
+### Best-effort resize quota admission
+
+Validating webhook on `UPDATE`/`pods/resize` with old and proposed Pods. For
+KAI-scheduled Pods:
+
+1. Resolve PodGroup, preemptibility, leaf queue, and ancestors.
+2. Compute old and proposed effective requests (inherited `Infeasible` stale).
+3. `delta = max(proposed - old, 0)` per resource.
+4. Reject if any queue on the path would violate:
+
+```text
+all workloads:
+    Allocated + delta > limit
+
+non-preemptible:
+    AllocatedNotPreemptible + delta > quota (deserved)
+```
+
+Usage comes from the best-available snapshot (Queue status / live effective
+requests). Downsize always admissible; capacity frees only when effective
+usage falls. Limit-only changes with no request growth have zero delta.
+Rejections name Pod, PodGroup, limiting queue/ancestor, resource, allocation,
+delta, and boundary.
+
+This is **not** atomic across concurrent resizes or against in-flight
+scheduler allocations. Existing violations (race, bypass, pre-rollout) keep
+full effective accounting, are reported, and block further deepenings via the
+webhook and allocate-time capacity checks. No proactive "drain until under
+limit" action for now.
+
+### Optional strict upsize block
+
+Config knob, **disabled by default**, rejects request upsizes that could race
+past a hard queue bound — even if current usage would still fit:
+
+- **Any** upsize, if the leaf queue or an ancestor has a finite `limit` on that
+  resource.
+- **Non-preemptible** upsize, if the leaf queue or an ancestor has a finite
+  `quota` (deserved) on that resource.
+
+Escape hatch for operators who cannot tolerate best-effort races without a
+ledger. Preemptible upsizes on queues that are only bounded by deserved (no
+finite limit) remain subject to the best-effort check only — they may go over
+deserved by design.
+
+```yaml
+spec:
+  admission:
+    inPlacePodResize:
+      validateQuota: true                 # best-effort hierarchical checks
+      blockUpsizeOnBoundedQueues: false   # conservative; default off
+```
+
+Exact CR field names TBD at implementation.
+
+## Decisions
+
+| Topic | Decision |
+| --- | --- |
+| Accounting model | Upstream effective request; upstream reason-only `Infeasible` semantics (kubelet owns the condition lifecycle) |
+| Resize admit path | Keep native `pods/resize`; validate in webhook (do **not** convert to a KAI-owned scheduling API — breaks VPA / API contract) |
+| Non-preemptible growth past quota | Reject at webhook; keep reject at allocate |
+| Concurrent-resize / scheduler races | Best effort; optional `blockUpsizeOnBoundedQueues`; reservation ledger only if a concrete issue appears |
+| Drain-until-under-limit action | Rejected — poor UX; reclaim stays demand-driven |
+| Deferred resize preemption | Separate track: [#1872](https://github.com/kai-scheduler/KAI-Scheduler/issues/1872) / [#2051](https://github.com/kai-scheduler/KAI-Scheduler/pull/2051). Depends on `Deferred` charged at max(spec, actual) — the reserved target is its thrash safety |
+| Wait for upstream resize gates / KEP-5836 only | Rejected as sole plan — ship Goals accounting + admit now; upstream remains complementary ([kubernetes#131835](https://github.com/kubernetes/kubernetes/issues/131835)) |
+
+## Known gaps
+
+Best-effort admission can race: concurrent upsizes, or a resize vs scheduler
+allocate, may double-spend the same remaining headroom.
+`blockUpsizeOnBoundedQueues` and allocate/webhook deepenings mitigate today.
+Prefer upstream resize gates
+([kubernetes#131835](https://github.com/kubernetes/kubernetes/issues/131835))
+as the long-term out-of-tree quota hook; add a reservation ledger only if
+gates do not land and races hurt in production.
+
+## References
+
+- [KEP-1287: In-place Update of Pod Resources](https://github.com/kubernetes/enhancements/blob/master/keps/sig-node/1287-in-place-update-pod-resources/README.md)
+- [Kubernetes 1.35 in-place container resize](https://v1-35.docs.kubernetes.io/docs/tasks/configure-pod-container/resize-container-resources/)
+- [Upstream effective Pod request helper](https://github.com/kubernetes/kubernetes/blob/v1.35.4/staging/src/k8s.io/component-helpers/resource/helpers.go)
+- [Upstream Pod quota evaluator](https://github.com/kubernetes/kubernetes/blob/v1.35.4/pkg/quota/v1/evaluator/core/pods.go)
+- [KEP-5836: Scheduler Preemption for In-Place Pod Resize](https://www.kubernetes.dev/resources/keps/5836)
+- [kubernetes#131835: out-of-tree quota vs in-place resize](https://github.com/kubernetes/kubernetes/issues/131835)