fix(scheduler,host): sort numa node by numa distance check memsize (#23923)

Author: wanyaoqi

cmd/climc/shell/compute/servers.go

@@ -149,6 +149,7 @@ func init() {
 	cmd.Get("qemu-info", new(options.ServerIdOptions))
 	cmd.Get("hardware-info", new(options.ServerIdOptions))
 	cmd.Get("screen-dump-show", new(options.ServerScreenDumpOptions))
+	cmd.Get("numa-info", new(options.ServerIdOptions))
 	cmd.BatchPerform("screen-dump", new(options.ServerIdsOptions))
 
 	cmd.GetProperty(&options.ServerStatusStatisticsOptions{})

pkg/apis/compute/guest_const.go

@@ -256,6 +256,11 @@ const (
 	CPU_MODE_HOST = "host"
 )
 
+const (
+	VM_CPU_NUMA_PIN_SCHEDULER = "scheduler"
+	VM_CPU_NUMA_PIN_HOST      = "host"
+)
+
 const (
 	VM_MACHINE_TYPE_PC  = "pc"
 	VM_MACHINE_TYPE_Q35 = "q35"
@@ -345,7 +350,9 @@ const (
 
 	VM_METADATA_RELEASED_DEVICES = "released_devices"
 
-	VM_METADATA_CPU_NUMA_PIN                   = "__cpu_numa_pin"
+	VM_METADATA_CPU_NUMA_PIN      = "__cpu_numa_pin"
+	VM_METADATA_CPU_NUMA_PIN_TYPE = "__cpu_numa_pin_type"
+
 	VM_METADATA_VIRTUAL_ISOLATED_DEVICE_CONFIG = "__virtual_isolated_device_config"
 	// Kickstart相关metadata常量
 	VM_METADATA_KICKSTART_CONFIG         = "kickstart_config"

pkg/apis/compute/guests.go

@@ -1111,6 +1111,13 @@ type ServerGetCPUSetCoresResp struct {
 	HostUsedCores []int `json:"host_used_cores"`
 }
 
+type ServerGetNumaInfoInput struct{}
+
+type ServerGetNumaInfoResp struct {
+	CpuNumaPin              jsonutils.JSONObject
+	IsolatedDevicesNumaNode []int8
+}
+
 type ServerGetHardwareInfoInput struct{}
 
 type ServerHardwareInfoMotherboard struct {

pkg/compute/models/guest_actions.go

@@ -560,7 +560,7 @@ func (self *SGuest) GetSchedMigrateParams(
 			schedDesc.SkipKernelCheck = &input.SkipKernelCheck
 			schedDesc.HostMemPageSizeKB = host.PageSizeKB
 		}
-		if self.CpuNumaPin != nil {
+		if self.IsSchedulerNumaAllocate() {
 			cpuNumaPin := make([]schedapi.SCpuNumaPin, 0)
 			self.CpuNumaPin.Unmarshal(&cpuNumaPin)
 			schedDesc.CpuNumaPin = cpuNumaPin
@@ -1738,7 +1738,7 @@ func (self *SGuest) StartGueststartTask(
 		}
 	}
 
-	if !startFromCreate && self.CpuNumaPin != nil {
+	if !startFromCreate && self.IsSchedulerNumaAllocate() {
 		// clean cpu numa pin
 		err := self.SetCpuNumaPin(ctx, userCred, nil, nil)
 		if err != nil {
@@ -1783,7 +1783,7 @@ func (self *SGuest) GuestNonSchedStartTask(
 	if self.BackupHostId != "" {
 		taskName = "HAGuestStartTask"
 	}
-	if self.CpuNumaPin != nil {
+	if self.IsSchedulerNumaAllocate() {
 		srcSchedCpuNumaPin := make([]schedapi.SCpuNumaPin, 0)
 		err := self.CpuNumaPin.Unmarshal(&srcSchedCpuNumaPin)
 		if err != nil {
@@ -6788,6 +6788,22 @@ func (self *SGuest) GetDetailsCpusetCores(ctx context.Context, userCred mcclient
 	return resp, nil
 }
 
+func (self *SGuest) GetDetailsNumaInfo(ctx context.Context, userCred mcclient.TokenCredential, _ *api.ServerGetNumaInfoInput) (*api.ServerGetNumaInfoResp, error) {
+	ret := new(api.ServerGetNumaInfoResp)
+	devs, _ := self.GetIsolatedDevices()
+	if len(devs) > 0 {
+		ret.IsolatedDevicesNumaNode = make([]int8, 0)
+		for i := range devs {
+			if devs[i].NumaNode > 0 {
+				ret.IsolatedDevicesNumaNode = append(ret.IsolatedDevicesNumaNode, devs[i].NumaNode)
+			}
+		}
+	}
+
+	ret.CpuNumaPin = self.CpuNumaPin
+	return ret, nil
+}
+
 func (self *SGuest) GetDetailsHardwareInfo(ctx context.Context, userCred mcclient.TokenCredential, _ *api.ServerGetHardwareInfoInput) (*api.ServerGetHardwareInfoResp, error) {
 	host, err := self.GetHost()
 	if err != nil {

pkg/compute/models/guests.go

@@ -1394,9 +1394,13 @@ func (guest *SGuest) SetCpuNumaPin(
 		}
 	}
 
+	var cpuNumaPinType string
 	var schedCpuNumaPinJ jsonutils.JSONObject
 	if schedCpuNumaPin != nil {
 		schedCpuNumaPinJ = jsonutils.Marshal(schedCpuNumaPin)
+		cpuNumaPinType = api.VM_CPU_NUMA_PIN_SCHEDULER
+	} else if cpuNumaPin != nil {
+		schedCpuNumaPinJ = jsonutils.Marshal(cpuNumaPin)
 	}
 	diff, err := db.Update(guest, func() error {
 		guest.CpuNumaPin = schedCpuNumaPinJ
@@ -1410,7 +1414,11 @@ func (guest *SGuest) SetCpuNumaPin(
 	if cpuNumaPin != nil {
 		jcpuNumaPin = jsonutils.Marshal(cpuNumaPin)
 	}
-	err = guest.SetMetadata(ctx, api.VM_METADATA_CPU_NUMA_PIN, jcpuNumaPin, userCred)
+	metadataMap := map[string]interface{}{
+		api.VM_METADATA_CPU_NUMA_PIN:      jcpuNumaPin,
+		api.VM_METADATA_CPU_NUMA_PIN_TYPE: cpuNumaPinType,
+	}
+	err = guest.SetAllMetadata(ctx, metadataMap, userCred)
 	if err != nil {
 		return err
 	}
@@ -5003,7 +5011,7 @@ func (self *SGuest) createDiskOnHost(
 
 func (self *SGuest) CreateIsolatedDeviceOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, devs []*api.IsolatedDeviceConfig, pendingUsage quotas.IQuota) error {
 	var numaNodes []int
-	if self.CpuNumaPin != nil {
+	if self.IsSchedulerNumaAllocate() {
 		numaNodes = make([]int, 0)
 		cpuNumaPin := make([]schedapi.SCpuNumaPin, 0)
 		self.CpuNumaPin.Unmarshal(&cpuNumaPin)
@@ -5296,6 +5304,11 @@ func (self *SGuest) isNeedDoResetPasswd() bool {
 	return true
 }
 
+func (self *SGuest) IsSchedulerNumaAllocate() bool {
+	cpuNumaPinType := self.GetMetadata(context.Background(), api.VM_METADATA_CPU_NUMA_PIN_TYPE, nil)
+	return cpuNumaPinType == api.VM_CPU_NUMA_PIN_SCHEDULER && self.CpuNumaPin != nil
+}
+
 func (self *SGuest) GetDeployConfigOnHost(ctx context.Context, userCred mcclient.TokenCredential, host *SHost, params *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
 	config := jsonutils.NewDict()
 
@@ -5563,7 +5576,7 @@ func (self *SGuest) GetJsonDescAtHypervisor(ctx context.Context, host *SHost) *a
 		desc.IsolatedDevices = append(desc.IsolatedDevices, dev.getDesc())
 	}
 
-	if self.CpuNumaPin != nil {
+	if self.IsSchedulerNumaAllocate() {
 		cpuNumaPin := make([]api.SCpuNumaPin, 0)
 		cpuNumaPinStr := self.GetMetadata(ctx, api.VM_METADATA_CPU_NUMA_PIN, nil)
 		cpuNumaPinJson, err := jsonutils.ParseString(cpuNumaPinStr)

pkg/compute/models/hosts.go

@@ -156,7 +156,8 @@ type SHost struct {
 	// 内存超分比
 	MemCmtbound float32 `nullable:"true" list:"domain" create:"domain_optional"`
 	// 页大小
-	PageSizeKB         int  `nullable:"false" default:"4" list:"domain" update:"domain" create:"domain_optional"`
+	PageSizeKB int `nullable:"false" default:"4" list:"domain" update:"domain" create:"domain_optional"`
+	// scheduler cpu-node/numa allocate
 	EnableNumaAllocate bool `nullable:"true" default:"false" list:"domain" update:"domain" create:"domain_optional"`
 
 	// 存储大小,单位Mb

pkg/compute/tasks/guest/guest_start_task.go

@@ -112,6 +112,19 @@ func (task *GuestStartTask) OnStartComplete(ctx context.Context, obj db.IStandal
 	guest.SetMetadata(ctx, api.VM_METADATA_START_VMEM_MB, guest.VmemSize, task.UserCred)
 	// save start time
 	guest.SaveLastStartAt()
+
+	if data.Contains("cpu_numa_pin") {
+		cpuNumaPin := make([]api.SCpuNumaPin, 0)
+		err := data.Unmarshal(&cpuNumaPin, "cpu_numa_pin")
+		if err != nil {
+			log.Errorf("failed unmarshal cpu numa pin %s", err)
+		} else {
+			if err = guest.SetCpuNumaPin(ctx, task.UserCred, nil, cpuNumaPin); err != nil {
+				log.Errorf("failed set cpu numa pin %s", err)
+			}
+		}
+	}
+
 	// sync Vpc Topology
 	isVpc, err := guest.IsOneCloudVpcNetwork()
 	if err != nil {

pkg/hostman/guestman/guesthelper.go

@@ -374,7 +374,7 @@ func (pq *CpuSetCounter) AllocCpuset(vcpuCount int, memSizeKB int64, preferNumaN
 	pq.GuestIds[guestId] = struct{}{}
 
 	if pq.NumaEnabled && len(preferNumaNodes) > 0 {
-		sortedNumaDistance := pq.getDistancesSeqByPreferNodes(preferNumaNodes)
+		sortedNumaDistance := pq.getDistancesSeqByPreferNodes(preferNumaNodes, int(memSizeKB))
 		for nodeCount := 1; nodeCount <= len(pq.Nodes); nodeCount *= 2 {
 			ret := pq.allocCpuNumaNodesByPreferNodes(vcpuCount, int(memSizeKB), nodeCount, sortedNumaDistance)
 			if ret != nil {
@@ -459,7 +459,7 @@ type SSortedNumaDistance struct {
 	CpuReserved bool
 }
 
-func (pq *CpuSetCounter) getDistancesSeqByPreferNodes(preferNumaNodes []int8) []SSortedNumaDistance {
+func (pq *CpuSetCounter) getDistancesSeqByPreferNodes(preferNumaNodes []int8, memSizeKB int) []SSortedNumaDistance {
 	sortedNumaDistance := make([]SSortedNumaDistance, len(pq.Nodes))
 	for i := range pq.Nodes {
 		distance := 0
@@ -496,7 +496,11 @@ func (pq *CpuSetCounter) getDistancesSeqByPreferNodes(preferNumaNodes []int8) []
 		if sortedNumaDistance[i].CpuReserved {
 			return sortedNumaDistance[i].UsedRate < sortedNumaDistance[j].UsedRate
 		}
-		return sortedNumaDistance[i].FreeMemSize > sortedNumaDistance[j].FreeMemSize
+		if sortedNumaDistance[i].Distance < sortedNumaDistance[j].Distance {
+			return sortedNumaDistance[i].FreeMemSize > memSizeKB && sortedNumaDistance[j].FreeMemSize-sortedNumaDistance[i].FreeMemSize <= 2*memSizeKB
+		} else {
+			return sortedNumaDistance[j].FreeMemSize > memSizeKB && sortedNumaDistance[i].FreeMemSize-sortedNumaDistance[j].FreeMemSize >= 2*memSizeKB
+		}
 	})
 	return sortedNumaDistance
 }

pkg/hostman/guestman/guestman.go

@@ -113,9 +113,9 @@ type SGuestManager struct {
 	qemuMachineCpuMax map[string]uint
 	qemuMaxMem        int
 
-	numaAllocate bool
-	cpuSet       *CpuSetCounter
-	pythonPath   string
+	hostagentNumaAllocate bool
+	cpuSet                *CpuSetCounter
+	pythonPath            string
 
 	// container related members
 	containerProbeManager      prober.Manager
@@ -344,7 +344,7 @@ func (m *SGuestManager) Bootstrap() (chan struct{}, error) {
 
 	if options.HostOptions.EnableHostAgentNumaAllocate {
 		enableMemAlloc := m.host.IsContainerHost() || m.host.IsHugepagesEnabled()
-		m.numaAllocate = !m.host.IsNumaAllocateEnabled() && enableMemAlloc && (len(hostTypo.Nodes) > 1)
+		m.hostagentNumaAllocate = !m.host.IsSchedulerNumaAllocateEnabled() && enableMemAlloc && (len(hostTypo.Nodes) > 1)
 	}
 
 	var reserveCpus = cpuset.NewCPUSet()
@@ -357,7 +357,7 @@ func (m *SGuestManager) Bootstrap() (chan struct{}, error) {
 	}
 
 	cpuSet, err := NewGuestCpuSetCounter(
-		hostTypo, reserveCpus, m.numaAllocate, m.host.IsContainerHost(),
+		hostTypo, reserveCpus, m.hostagentNumaAllocate, m.host.IsContainerHost(),
 		m.host.HugepageSizeKb(), m.host.CpuCmtBound(), m.host.MemCmtBound(), m.host.GetReservedMemMb(),
 	)
 	if err != nil {

pkg/hostman/guestman/guesttasks.go

@@ -2841,7 +2841,7 @@ func (task *SGuestHotplugCpuMemTask) updateGuestDesc() {
 		}
 		task.Desc.MemDesc.MemSlots = append(task.Desc.MemDesc.MemSlots, task.memSlots...)
 
-		if task.manager.numaAllocate {
+		if task.manager.hostagentNumaAllocate {
 			for i := range task.memSlotNewIndexs {
 				hugepageId := fmt.Sprintf("%s-%d", task.getOriginId(), task.memSlotNewIndexs[i])
 				task.validateNumaAllocated(hugepageId, false, true, nil)