Skip to content

libvirt: add cpuset support for CPU pinning #2744

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
tak-ka3 wants to merge 1 commit into
base: main
Choose a base branch
from
+115 −5
Open

libvirt: add cpuset support for CPU pinning #2744

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions src/cloud-api-adaptor/install/overlays/libvirt/kustomization.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ configMapGenerator:
#- LIBVIRT_VOL_NAME="" # Uncomment and set if you want to use a specific volume name. Defaults to podvm-base.qcow2
#- LIBVIRT_CPU="2"
#- LIBVIRT_MEMORY="8192"
#- LIBVIRT_CPUSET="" # Uncomment and set to pin vCPUs to physical CPUs (e.g., "0,2,4,6" or "0-3")
Copy link
Contributor

@ajaypvictor ajaypvictor Jan 7, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

can we have an option to provide this as an annotation as well? similar to how we could provide vcpus as io.katacontainers.config.hypervisor.default_vcpus?

Copy link
Member

@stevenhorsman stevenhorsman Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not sure how do-able this is which our current set-up - the annotation are provided via the kata runtime and having a new annotation there for a single remote cloud-provider doesn't feel like the right approach, unless it can be transparently passed through?

Copy link
Contributor

@ajaypvictor ajaypvictor Jan 7, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thanks steve for the clarification, was wondering if there was a similar or equivalent annotation from kata for cpusets which we could leverage similar to the image annotation (I agree that it's not a cleaner way).

#- PAUSE_IMAGE="" # Uncomment and set if you want to use a specific pause image
#- TUNNEL_TYPE="" # Uncomment and set if you want to use a specific tunnel type. Defaults to vxlan
#- VXLAN_PORT="" # Uncomment and set if you want to use a specific vxlan port. Defaults to 4789
Expand Down
7 changes: 4 additions & 3 deletions src/cloud-providers/libvirt/libvirt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -250,7 +250,8 @@ func createDomainXMLs390x(client *libvirtClient, cfg *domainConfig, vm *vmConfig
Value: cfg.mem, Unit: "MiB",
},
VCPU: &libvirtxml.DomainVCPU{
Value: cfg.cpu,
Value: cfg.cpu,
CPUSet: vm.cpuset,
},
Clock: &libvirtxml.DomainClock{
Offset: "utc",
Expand Down Expand Up @@ -309,7 +310,7 @@ func createDomainXMLx86_64(client *libvirtClient, cfg *domainConfig, vm *vmConfi
Name: cfg.name,
Description: "This Virtual Machine is the peer-pod VM",
Memory: &libvirtxml.DomainMemory{Value: cfg.mem, Unit: "MiB", DumpCore: "on"},
VCPU: &libvirtxml.DomainVCPU{Value: cfg.cpu},
VCPU: &libvirtxml.DomainVCPU{Value: cfg.cpu, CPUSet: vm.cpuset},
OS: &libvirtxml.DomainOS{
Type: &libvirtxml.DomainOSType{Arch: "x86_64", Type: typeHardwareVirtualMachine},
},
Expand Down Expand Up @@ -459,7 +460,7 @@ func createDomainXMLaarch64(client *libvirtClient, cfg *domainConfig, vm *vmConf
Firmware: "efi",
},
Memory: &libvirtxml.DomainMemory{Value: cfg.mem, Unit: "MiB"},
VCPU: &libvirtxml.DomainVCPU{Value: cfg.cpu},
VCPU: &libvirtxml.DomainVCPU{Value: cfg.cpu, CPUSet: vm.cpuset},
CPU: &libvirtxml.DomainCPU{Mode: "host-passthrough"},
Devices: &libvirtxml.DomainDeviceList{
Disks: []libvirtxml.DomainDisk{
Expand Down
105 changes: 105 additions & 0 deletions src/cloud-providers/libvirt/libvirt_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -152,3 +152,108 @@ func TestCreateDomainXMLaarch64(t *testing.T) {
t.Error(err)
}
}

func TestCPUSetInDomainXML(t *testing.T) {
checkConfig(t)

client, err := NewLibvirtClient(testCfg)
if err != nil {
t.Fatal(err)
}
defer client.connection.Close()

testCases := []struct {
name string
cpuset string
expectedCPUSet string
}{
{
name: "with cpuset specified",
cpuset: "0,2,4,6",
expectedCPUSet: "0,2,4,6",
},
{
name: "with cpuset range",
cpuset: "0-3",
expectedCPUSet: "0-3",
},
{
name: "with empty cpuset",
cpuset: "",
expectedCPUSet: "",
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
vm := vmConfig{
cpuset: tc.cpuset,
}

domainCfg := domainConfig{
name: "TestCPUSet",
cpu: 4,
mem: 4096,
networkName: client.networkName,
bootDisk: "/var/lib/libvirt/images/root.qcow2",
cidataDisk: "/var/lib/libvirt/images/cidata.iso",
}

domCfg, err := createDomainXML(client, &domainCfg, &vm)
if err != nil {
t.Error(err)
return
}

assert.Equal(t, tc.expectedCPUSet, domCfg.VCPU.CPUSet,
"CPUSet should be set correctly in domain XML")
})
}
}

func TestCPUSetXMLMarshaling(t *testing.T) {
// Test that CPUSet is correctly marshaled to XML without requiring libvirt connection
testCases := []struct {
name string
cpuset string
expectedSubstr string
}{
{
name: "cpuset with comma-separated values",
cpuset: "0,2,4,6",
expectedSubstr: `cpuset="0,2,4,6"`,
},
{
name: "cpuset with range",
cpuset: "0-7",
expectedSubstr: `cpuset="0-7"`,
},
{
name: "cpuset with mixed format",
cpuset: "0,2,4-7,10",
expectedSubstr: `cpuset="0,2,4-7,10"`,
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
domain := &libvirtxml.Domain{
Type: "kvm",
Name: "test-vm",
VCPU: &libvirtxml.DomainVCPU{
Value: 4,
CPUSet: tc.cpuset,
},
}

xmlStr, err := domain.Marshal()
if err != nil {
t.Errorf("Failed to marshal domain XML: %v", err)
return
}

assert.Contains(t, xmlStr, tc.expectedSubstr,
"Marshaled XML should contain cpuset attribute")
})
}
}
1 change: 1 addition & 0 deletions src/cloud-providers/libvirt/manager.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ func (_ *Manager) ParseCmd(flags *flag.FlagSet) {
reg.StringWithEnv(&libvirtcfg.Firmware, "firmware", defaultFirmware, "LIBVIRT_EFI_FIRMWARE", "Path to OVMF")
reg.UintWithEnv(&libvirtcfg.CPU, "cpu", 2, "LIBVIRT_CPU", "Number of processors allocated")
reg.UintWithEnv(&libvirtcfg.Memory, "memory", 8192, "LIBVIRT_MEMORY", "Amount of memory in MiB")
reg.StringWithEnv(&libvirtcfg.CPUSet, "cpuset", "", "LIBVIRT_CPUSET", "CPU set for pinning vCPUs to physical CPUs (e.g., \"0,2,4,6\" or \"0-3\")")

// Flags without environment variable support (pass empty string for envVarName)
reg.StringWithEnv(&libvirtcfg.DataDir, "data-dir", defaultDataDir, "", "libvirt storage dir")
Expand Down
2 changes: 1 addition & 1 deletion src/cloud-providers/libvirt/provider.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ func (p *libvirtProvider) CreateInstance(ctx context.Context, podName, sandboxID
}

// TODO: Specify the maximum instance name length in Libvirt
vm := &vmConfig{name: instanceName, cpu: instanceVCPUs, mem: instanceMemory, userData: userData, firmware: p.serviceConfig.Firmware}
vm := &vmConfig{name: instanceName, cpu: instanceVCPUs, mem: instanceMemory, userData: userData, firmware: p.serviceConfig.Firmware, cpuset: p.serviceConfig.CPUSet}

if p.serviceConfig.DisableCVM {
vm.launchSecurityType = NoLaunchSecurity
Expand Down
4 changes: 3 additions & 1 deletion src/cloud-providers/libvirt/types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,8 @@ type Config struct {
LaunchSecurity string
Firmware string
CPU uint
Memory uint // It stores the value in MiB
Memory uint // It stores the value in MiB
CPUSet string // CPU set for pinning vCPUs (e.g., "0,2,4,6" or "0-3")
}

type vmConfig struct {
Expand All @@ -35,6 +36,7 @@ type vmConfig struct {
instanceId string //keeping it consistent with sandbox.vsi
launchSecurityType LaunchSecurityType
firmware string
cpuset string // CPU set for pinning vCPUs (e.g., "0,2,4,6" or "0-3")
}

type createDomainOutput struct {
Expand Down
Loading