Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 15 additions & 4 deletions pkg/apis/devices.harvesterhci.io/v1beta1/pcideviceclaim.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,10 @@ type PCIDeviceClaim struct {
}

type PCIDeviceClaimSpec struct {
Address string `json:"address"`
NodeName string `json:"nodeName"`
UserName string `json:"userName"`
Address string `json:"address"`
NodeName string `json:"nodeName"`
UserName string `json:"userName"`
DisableResourcePooling bool `json:"disableResourcePooling,omitempty"`
Comment on lines +24 to +27
}

func (s PCIDeviceClaimSpec) NodeAddr() string {
Expand All @@ -36,6 +37,16 @@ type PCIDeviceClaimStatus struct {
}

const (
SkipVFIOBindingAnnotationKey = "pcidevices.harvesterhci.io/skip-vfio-binding"
SkipVFIOBindingAnnotationKey = "pcidevices.harvesterhci.io/skip-vfio-binding"

// PCIDeviceOverrideResourceName is an annotation key shared by two flows:
// 1. vGPU: set on both PCIDeviceClaim and PCIDevice by the vgpu controller;
// lifecycle (add/remove) is fully managed by the vgpu controller.
// 2. Individual PCIDevice (DisableResourcePooling): set on PCIDevice by the
// pcideviceclaim controller (ensureIndividualDeviceResourceName) when the
// claim is created, and removed by the pcideviceclaim controller
// (cleanupIndividualDeviceResourceName) when the claim is deleted.
// In both cases the pcidevice controller reads this annotation to propagate
// the override value into PCIDevice.Status.ResourceName on each reconcile cycle.
PCIDeviceOverrideResourceName = "pcidevice.harvesterhci.io/override-resource-name"
)
9 changes: 6 additions & 3 deletions pkg/controller/pcidevice/pcidevice_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -88,9 +88,12 @@ func (h *Handler) ReconcilePCIDevices(nodename string) error {
}

devCopy := devCR.DeepCopy()
// needed for vgpu operation. This annotation will be added and removed by
// the vgpu controller on pcidevice object as part of enabling/disable a
// pcidevice
// PCIDeviceOverrideResourceName is used by two flows:
// - vGPU: the vgpu controller sets/removes this annotation on PCIDevice
// to override the resource name with the vGPU profile name.
// - Individual PCIDevice: the pcideviceclaim controller sets/removes this
// annotation with a stable name when DisableResourcePooling is enabled.
// When present, the value takes precedence over the auto-generated resource name.
overrideResourceName := devCopy.Annotations[v1beta1.PCIDeviceOverrideResourceName]
// during reboot if the device driver has changed back from vfio, then update the CRD
// to correct driver in use. This will ensure that the original driver is correctly updated on device
Expand Down
69 changes: 69 additions & 0 deletions pkg/controller/pcideviceclaim/pcideviceclaim_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,19 @@ func (h *Handler) OnRemove(_ string, pdc *v1beta1.PCIDeviceClaim) (*v1beta1.PCID
delete(h.devicePlugins, resourceName)
}
}

// For DisableResourcePooling claims, remove the per-device override annotation that was
// set on PCIDevice by ensureIndividualDeviceResourceName during claim creation.
// Since PCIDeviceClaim is immutable (create/delete only), pdc.Spec.DisableResourcePooling
// at deletion time is always the same value as at creation time.
// Note: vGPU claims do NOT set DisableResourcePooling; their annotation lifecycle is
// managed entirely by the vgpu controller.
if pdc.Spec.DisableResourcePooling {
if err := h.cleanupIndividualDeviceResourceName(pd); err != nil {
return pdc, fmt.Errorf("error cleaning up individual device resource name: %v", err)
}
}

return pdc, nil
}

Expand Down Expand Up @@ -323,6 +336,52 @@ func getOrphanedPCIDevices(
return &pdsOrphaned, nil
}

// ensureIndividualDeviceResourceName sets a unique resource name (<vendor-prefix>/<pd.Name>)
// on the PCIDevice annotation and Status.ResourceName when DisableResourcePooling is enabled.
// Cleanup is handled by cleanupIndividualDeviceResourceName on claim deletion.
func (h *Handler) ensureIndividualDeviceResourceName(pd *v1beta1.PCIDevice) (*v1beta1.PCIDevice, error) {
parts := strings.SplitN(pd.Status.ResourceName, "/", 2)
individualResourceName := pd.Name
if len(parts) == 2 {
individualResourceName = parts[0] + "/" + pd.Name
}
Comment on lines +339 to +347

pdCopy := pd.DeepCopy()
if pdCopy.Annotations == nil {
pdCopy.Annotations = make(map[string]string)
}
pdCopy.Annotations[v1beta1.PCIDeviceOverrideResourceName] = individualResourceName
updated, err := h.pdClient.Update(pdCopy)
if err != nil {
return pd, err
}

updated.Status.ResourceName = individualResourceName
updated, err = h.pdClient.UpdateStatus(updated)
if err != nil {
return updated, err
}

Comment on lines +349 to +364
return updated, nil
}

// cleanupIndividualDeviceResourceName removes the PCIDeviceOverrideResourceName annotation from the
// PCIDevice when a DisableResourcePooling claim is deleted. Once the annotation is gone, the pcidevice
// controller will revert Status.ResourceName to the auto-generated value on its next reconcile cycle.
// Only called for DisableResourcePooling claims — vGPU annotation cleanup is handled by the vgpu controller.
func (h *Handler) cleanupIndividualDeviceResourceName(pd *v1beta1.PCIDevice) error {
if pd.Annotations == nil {
return nil
}
if _, ok := pd.Annotations[v1beta1.PCIDeviceOverrideResourceName]; !ok {
return nil
}
pdCopy := pd.DeepCopy()
delete(pdCopy.Annotations, v1beta1.PCIDeviceOverrideResourceName)
_, err := h.pdClient.Update(pdCopy)
return err
}

func checkGroupMap(pd *v1beta1.PCIDevice) error {
group, err := iommu.GetGroupMap(pd.Status.Address)
if err != nil {
Expand Down Expand Up @@ -362,6 +421,16 @@ func (h *Handler) reconcilePCIDeviceClaims(_ string, pdc *v1beta1.PCIDeviceClaim
return pdc, err
}

// When DisableResourcePooling is set, persist the per-device resource name on the
// PCIDevice object so that the frontend (and any other consumer) can read
// pd.Status.ResourceName uniformly without needing claim-level awareness.
if pdc.Spec.DisableResourcePooling {
pd, err = h.ensureIndividualDeviceResourceName(pd)
if err != nil {
return pdc, fmt.Errorf("error setting individual device resource name: %v", err)
}
}
Comment on lines +424 to +432

lock.Lock()
defer lock.Unlock()

Expand Down
Loading