feat: support DRA admin access in the device allocator

Author: yindia

pkg/controllers/dynamicresources/deviceallocation/controller.go

@@ -156,6 +156,12 @@ func (c *Controller) reconcileClaim(ctx context.Context, nn types.NamespacedName
 	contributions := make(map[cloudprovider.DeviceID]DeviceContribution, len(claim.Status.Allocation.Devices.Results))
 	for i := range claim.Status.Allocation.Devices.Results {
 		result := &claim.Status.Allocation.Devices.Results[i]
+		// Admin-access allocations bind for privileged monitoring only; they don't consume
+		// the device and must not mark it allocated, so other claims (and Karpenter's
+		// scheduling simulation) still treat it as available (KEP-5018).
+		if lo.FromPtr(result.AdminAccess) {
+			continue
+		}
 		deviceID := cloudprovider.DeviceID{
 			Driver: unique.Make(result.Driver),
 			Pool:   unique.Make(result.Pool),

pkg/controllers/dynamicresources/deviceallocation/suite_test.go

@@ -25,6 +25,7 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	resourcev1 "k8s.io/api/resource/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -138,6 +139,15 @@ func deviceResult(device string) resourcev1.DeviceRequestAllocationResult {
 	}
 }
 
+// deviceResultAdmin builds an allocation result marked adminAccess, which binds for monitoring only
+// and must not be tracked as consuming the device (KEP-5018).
+func deviceResultAdmin(device string) resourcev1.DeviceRequestAllocationResult {
+	r := deviceResult(device)
+	adminAccess := true
+	r.AdminAccess = &adminAccess
+	return r
+}
+
 // deviceID constructs a cloudprovider.DeviceID using interned handles, matching the controller's representation.
 func deviceID(device string) cloudprovider.DeviceID {
 	return cloudprovider.DeviceID{
@@ -263,6 +273,41 @@ var _ = Describe("DeviceAllocation Controller", func() {
 		})
 	})
 
+	Describe("Admin access (KEP-5018)", func() {
+		BeforeEach(func() {
+			// The apiserver rejects adminAccess allocations unless the namespace carries the label.
+			ns := &corev1.Namespace{}
+			Expect(env.Client.Get(ctx, client.ObjectKey{Name: "default"}, ns)).To(Succeed())
+			if ns.Labels == nil {
+				ns.Labels = map[string]string{}
+			}
+			ns.Labels["resource.kubernetes.io/admin-access"] = "true"
+			Expect(env.Client.Update(ctx, ns)).To(Succeed())
+			triggerHydration()
+		})
+		It("does not track a device allocated only for admin access", func() {
+			claim := resourceClaim("admin-claim", deviceResultAdmin("device-0"))
+			ExpectApplied(ctx, env.Client, claim)
+			ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(claim))
+
+			seq, err := controller.AllocatedDevices(ctx)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(collectDevices(seq)).To(BeEmpty())
+		})
+		It("tracks non-admin devices while ignoring admin-access results in the same claim", func() {
+			claim := resourceClaim("mixed-claim",
+				deviceResult("device-0"),
+				deviceResultAdmin("device-1"),
+			)
+			ExpectApplied(ctx, env.Client, claim)
+			ExpectReconcileSucceeded(ctx, controller, client.ObjectKeyFromObject(claim))
+
+			seq, err := controller.AllocatedDevices(ctx)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(collectDevices(seq)).To(Equal(expectedDevices(deviceID("device-0"))))
+		})
+	})
+
 	Describe("Basic allocation", func() {
 		BeforeEach(func() {
 			triggerHydration()

pkg/controllers/provisioning/dra_test.go

@@ -674,6 +674,66 @@ var _ = Describe("Dynamic Resource Allocation", func() {
 		})
 	})
 
+	Context("Admin access against existing nodes (Y)", func() {
+		// The apiserver rejects adminAccess claims unless the namespace carries the allowlist label.
+		BeforeEach(func() {
+			ns := &corev1.Namespace{}
+			Expect(env.Client.Get(ctx, client.ObjectKey{Name: "default"}, ns)).To(Succeed())
+			if ns.Labels == nil {
+				ns.Labels = map[string]string{}
+			}
+			ns.Labels["resource.kubernetes.io/admin-access"] = "true"
+			Expect(env.Client.Update(ctx, ns)).To(Succeed())
+		})
+
+		// heldDeviceNode creates an initialized gpu-it node publishing one in-cluster device that is already held by a
+		// live pod, so the device is tracked as allocated. Returns the node.
+		heldDeviceNode := func() *corev1.Node {
+			GinkgoHelper()
+			node := existingNode("gpu-it", true, corev1.ResourceList{
+				corev1.ResourceCPU: resource.MustParse("4"), corev1.ResourceMemory: resource.MustParse("4Gi"), corev1.ResourcePods: resource.MustParse("10"),
+			})
+			ExpectApplied(ctx, env.Client, nodeLocalSlice(node, gpuDriver, "incluster-gpu-0"))
+			livePod := test.Pod(test.PodOptions{ObjectMeta: metav1.ObjectMeta{Name: "live-pod"}})
+			ExpectApplied(ctx, env.Client, livePod)
+			ExpectApplied(ctx, env.Client, allocatedClusterWideClaim("held-claim",
+				test.NodeLocalPoolName(gpuDriver, node.Name), gpuDriver, "incluster-gpu-0", podConsumer(livePod)))
+			return node
+		}
+
+		It("should bind an admin-access claim to a held device on an existing node without launching a new node (Y1)", func() {
+			cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{gpuInstanceType("gpu-it", 1)}
+			ExpectApplied(ctx, env.Client, nodePool, test.DeviceClassWithSelector("gpu", gpuDriver))
+			node := heldDeviceNode()
+
+			ExpectApplied(ctx, env.Client, test.ResourceClaimForRequests("admin-claim", test.AdminExactDeviceRequest("req", "gpu", 1)))
+			pod := draPod("gpu", "admin-claim")
+			provisionDRA(pod)
+
+			// Admin access binds to the already-held in-cluster device, so the pod fits the existing node and no new
+			// NodeClaim is provisioned — the core ask of the issue.
+			scheduled := ExpectScheduled(ctx, env.Client, pod)
+			Expect(scheduled.Name).To(Equal(node.Name))
+			Expect(ExpectNodeClaims(ctx, env.Client)).To(HaveLen(0))
+		})
+
+		It("should launch a new node for a normal claim when the existing node's only device is held (Y2)", func() {
+			// Same setup as Y1 but with a normal (non-admin) claim: the held device is unavailable, so Karpenter must
+			// provision a new node. This is the contrast that proves admin access changed the outcome.
+			cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{gpuInstanceType("gpu-it", 1)}
+			ExpectApplied(ctx, env.Client, nodePool, test.DeviceClassWithSelector("gpu", gpuDriver))
+			node := heldDeviceNode()
+
+			ExpectApplied(ctx, env.Client, test.ResourceClaimForRequests("normal-claim", test.ExactDeviceRequest("req", "gpu", 1)))
+			pod := draPod("gpu", "normal-claim")
+			provisionDRA(pod)
+
+			scheduled := ExpectScheduled(ctx, env.Client, pod)
+			Expect(scheduled.Name).ToNot(Equal(node.Name))
+			Expect(ExpectNodeClaims(ctx, env.Client)).To(HaveLen(1))
+		})
+	})
+
 	Context("Topology propagation from node-local devices (D)", func() {
 		It("should tighten a new NodeClaim to the zone of a zoned in-cluster device (D1)", func() {
 			cloudProvider.InstanceTypes = []*cloudprovider.InstanceType{gpuInstanceType("gpu-it", 1)}

pkg/scheduling/dynamicresources/allocator.go

@@ -560,6 +560,9 @@ type deviceAllocationMetadata struct {
 	deviceWithID     DeviceWithID
 	consumedCapacity map[resourcev1.QualifiedName]resource.Quantity
 	requestName      RequestName
+	// adminAccess marks an allocation that binds for privileged monitoring without
+	// consuming the device (KEP-5018). Excluded from the committed device set.
+	adminAccess bool
 }
 
 // allocate runs a per-instance-type DFS over in-cluster and template devices.
@@ -633,10 +636,14 @@ func (a *allocator) allocate(instanceTypes []InstanceTypeID) (*AllocationResult,
 			a.allocatingCapacity = nil
 			a.templateAllocatingCapacity = nil
 
-			deviceIDsByIT[itID] = make([]DeviceID, len(a.allocatedDevicesMetadata))
+			deviceIDsByIT[itID] = make([]DeviceID, 0, len(a.allocatedDevicesMetadata))
 			itReqs := scheduling.NewRequirements()
-			for di, da := range a.allocatedDevicesMetadata {
-				deviceIDsByIT[itID][di] = da.deviceWithID.ID
+			for _, da := range a.allocatedDevicesMetadata {
+				// Admin-access devices are part of the solution but don't consume the device,
+				// so they're excluded from the set the tracker commits as allocated (KEP-5018).
+				if !da.adminAccess {
+					deviceIDsByIT[itID] = append(deviceIDsByIT[itID], da.deviceWithID.ID)
+				}
 				meta := claimAllocMeta[da.claimIndex]
 				// Update the contributed requirements for the device, each devices contributed requirements are intersected to
 				// find the contributed requirements for the instance type.
@@ -853,15 +860,22 @@ func (a *allocator) tryDevice(
 	deviceID := dw.ID
 
 	// 1. Availability check — multi-alloc devices use capacity as the gatekeeper;
-	//    exclusive devices use binary allocation tracking.
+	//    exclusive devices use binary allocation tracking. Admin-access requests bypass
+	//    both: they may bind to already-allocated devices and don't consume capacity
+	//    (KEP-5018). The in-DFS dedupe still applies so a request gets distinct devices.
 	var consumed map[resourcev1.QualifiedName]resource.Quantity
-	if dw.AllowMultipleAllocations {
+	switch {
+	case rd.AdminAccess:
+		if a.allocatedDevices.Has(deviceID) {
+			return false
+		}
+	case dw.AllowMultipleAllocations:
 		var ok bool
 		consumed, ok = a.checkCapacity(dw.Device, deviceID, rd)
 		if !ok {
 			return false
 		}
-	} else {
+	default:
 		if a.allocationTracker.IsAllocated(deviceID, a.nodeClaim, a.itID) {
 			return false
 		}
@@ -870,8 +884,9 @@ func (a *allocator) tryDevice(
 		}
 	}
 
-	// 2. Counter verification — check shared counter budgets.
-	if len(dw.ConsumesCounters) > 0 {
+	// 2. Counter verification — check shared counter budgets. Admin-access requests
+	//    ignore resource allocations, so counters are neither checked nor consumed.
+	if !rd.AdminAccess && len(dw.ConsumesCounters) > 0 {
 		poolKey := PoolKey{Driver: deviceID.Driver, Pool: deviceID.Pool}
 		var remainingCounterSets map[string]map[string]resourcev1.Counter
 		if deviceID.Template {
@@ -943,17 +958,22 @@ func (a *allocator) tryDevice(
 		deviceWithID:     dw,
 		consumedCapacity: consumed,
 		requestName:      rd.Name,
+		adminAccess:      rd.AdminAccess,
 	})
-	if dw.AllowMultipleAllocations {
-		// Ensures a multi-allocatable device has a allocating capacity map, even if it has no capacity dimensions.
-		// This is needed so that Commit() can identify multi-alloc devices via capacityConsumptionByIT presence.
-		allocatingCapacityMap := lo.Ternary(deviceID.Template, a.templateAllocatingCapacity, a.allocatingCapacity)
-		if allocatingCapacityMap[deviceID] == nil {
-			allocatingCapacityMap[deviceID] = make(map[resourcev1.QualifiedName]resource.Quantity)
+	// Admin-access allocations don't consume the device, so skip all capacity/counter
+	// bookkeeping (KEP-5018).
+	if !rd.AdminAccess {
+		if dw.AllowMultipleAllocations {
+			// Ensures a multi-allocatable device has a allocating capacity map, even if it has no capacity dimensions.
+			// This is needed so that Commit() can identify multi-alloc devices via capacityConsumptionByIT presence.
+			allocatingCapacityMap := lo.Ternary(deviceID.Template, a.templateAllocatingCapacity, a.allocatingCapacity)
+			if allocatingCapacityMap[deviceID] == nil {
+				allocatingCapacityMap[deviceID] = make(map[resourcev1.QualifiedName]resource.Quantity)
+			}
 		}
+		a.deductAllocatingCapacity(consumed, deviceID, deviceID.Template)
+		a.deductAllocatingCounters(dw.Device, PoolKey{Driver: deviceID.Driver, Pool: deviceID.Pool}, deviceID.Template)
 	}
-	a.deductAllocatingCapacity(consumed, deviceID, deviceID.Template)
-	a.deductAllocatingCounters(dw.Device, PoolKey{Driver: deviceID.Driver, Pool: deviceID.Pool}, deviceID.Template)
 
 	// Recurse.
 	if a.dfs(claimIdx, reqIdx, subReqIdx, slotIdx+1) {
@@ -962,8 +982,10 @@ func (a *allocator) tryDevice(
 
 	// Backtrack — undo in reverse order of application: capacity, counters, allocation, then
 	// requirements/pools, then constraints.
-	a.restoreAllocatingCapacity(consumed, deviceID, deviceID.Template)
-	a.restoreAllocatingCounters(dw.Device, PoolKey{Driver: deviceID.Driver, Pool: deviceID.Pool}, deviceID.Template)
+	if !rd.AdminAccess {
+		a.restoreAllocatingCapacity(consumed, deviceID, deviceID.Template)
+		a.restoreAllocatingCounters(dw.Device, PoolKey{Driver: deviceID.Driver, Pool: deviceID.Pool}, deviceID.Template)
+	}
 	a.allocatedDevicesMetadata = a.allocatedDevicesMetadata[:len(a.allocatedDevicesMetadata)-1]
 	a.allocatedDevices.Delete(deviceID)