feat(qrm): support set memory.high to throttle reclaimed_cores

pkg/agent/qrm-plugins/memory/dynamicpolicy/memoryadvisor/types.go

@@ -28,6 +28,7 @@ const (
 	ControlKnowKeyMemoryOffloading   MemoryControlKnobName = "memory_offloading"
 	ControlKnowKeyDyingMemcgReclaim  MemoryControlKnobName = "dying_memcg_reclaim"
 	ControlKnobKeyMemoryNUMAHeadroom MemoryControlKnobName = "memory_numa_headroom"
+	ControlKnobKeyMemoryHigh         MemoryControlKnobName = "memory_high"
 )
 
 const (

pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go

@@ -273,6 +273,8 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration
 
 	memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyMemoryLimitInBytes,
 		memoryadvisor.ControlKnobHandlerWithChecker(policyImplement.handleAdvisorMemoryLimitInBytes))
+	memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyMemoryHigh,
+		memoryadvisor.ControlKnobHandlerWithChecker(policyImplement.handleAdvisorMemoryHigh))
 	memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyCPUSetMems,
 		memoryadvisor.ControlKnobHandlerWithChecker(policyImplement.handleAdvisorCPUSetMems))
 	memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyDropCache,

pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go

@@ -26,6 +26,7 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/opencontainers/runc/libcontainer/cgroups"
 	"github.com/samber/lo"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/metadata"
@@ -392,6 +393,43 @@ func (p *DynamicPolicy) handleAdvisorResp(advisorResp *advisorsvc.ListAndWatchRe
 	return nil
 }
 
+func (p *DynamicPolicy) handleAdvisorMemoryHigh(
+	_ *config.Configuration,
+	_ interface{},
+	_ *dynamicconfig.DynamicAgentConfiguration,
+	emitter metrics.MetricEmitter,
+	metaServer *metaserver.MetaServer,
+	entryName, subEntryName string,
+	calculationInfo *advisorsvc.CalculationInfo, podResourceEntries state.PodResourceEntries,
+) error {
+	memoryHighStr := calculationInfo.CalculationResult.Values[string(memoryadvisor.ControlKnobKeyMemoryHigh)]
+	memoryHigh, err := strconv.ParseInt(memoryHighStr, 10, 64)
+	if err != nil {
+		return fmt.Errorf("parse %s: %s failed with error: %v", memoryadvisor.ControlKnobKeyMemoryHigh, memoryHighStr, err)
+	}
+
+	if !cgroups.IsCgroup2UnifiedMode() {
+		general.Infof("memory.high is not supported in cgroupv1 mode")
+		return nil
+	}
+
+	if calculationInfo.CgroupPath != "" {
+		if err = cgroupmgr.ApplyMemoryWithRelativePath(calculationInfo.CgroupPath, &common.MemoryData{
+			HighInBytes: memoryHigh,
+		}); err != nil {
+			return fmt.Errorf("apply memory.high failed with error: %v", err)
+		}
+
+		_ = emitter.StoreInt64(util.MetricNameMemoryHandleAdvisorMemoryHigh, memoryHigh,
+			metrics.MetricTypeNameRaw, metrics.ConvertMapToTags(map[string]string{
+				"cgroupPath": calculationInfo.CgroupPath,
+			})...)
+		return nil
+	}
+
+	return nil
+}
+
 func (p *DynamicPolicy) handleAdvisorMemoryLimitInBytes(
 	_ *config.Configuration,
 	_ interface{},

pkg/agent/qrm-plugins/util/consts.go

@@ -51,6 +51,7 @@ const (
 	MetricNameMemoryHandleAdvisorContainerEntryFailed = "memory_handle_advisor_container_entry_failed"
 	MetricNameMemoryHandleAdvisorExtraEntryFailed     = "memory_handle_advisor_extra_entry_failed"
 	MetricNameMemoryHandleAdvisorMemoryLimit          = "memory_handle_advisor_memory_limit"
+	MetricNameMemoryHandleAdvisorMemoryHigh           = "memory_handle_advisor_memory_high"
 	MetricNameMemoryHandleAdvisorDropCache            = "memory_handle_advisor_drop_cache"
 	MetricNameMemoryHandleAdvisorCPUSetMems           = "memory_handle_advisor_cpuset_mems"
 	MetricNameMemoryHandlerAdvisorMemoryOffload       = "memory_handler_advisor_memory_offloading"

pkg/agent/sysadvisor/plugin/qosaware/resource/memory/plugin/memory_guard.go

@@ -44,6 +44,9 @@ const (
 	reclaimMemoryUnlimited = -1
 
 	defaultProcZoneInfoFile = "/proc/zoneinfo"
+
+	// Factor for calculating memory.high based on memory.max, available for cgroup v2 only
+	memoryHighScaleFactor = 0.95
 )
 
 type memoryGuard struct {
@@ -110,11 +113,19 @@ func (mg *memoryGuard) GetAdvices() types.InternalMemoryCalculationResult {
 		general.Errorf("failed to get last reconcile result")
 		return types.InternalMemoryCalculationResult{}
 	}
+	memoryMax := mg.reclaimMemoryLimit.Load()
+	memoryHigh := int64(float64(memoryMax) * memoryHighScaleFactor)
+	if memoryMax == reclaimMemoryUnlimited {
+		memoryHigh = reclaimMemoryUnlimited
+	}
 	result := types.InternalMemoryCalculationResult{
 		ExtraEntries: []types.ExtraMemoryAdvices{
 			{
 				CgroupPath: mg.reclaimRelativeRootCgroupPath,
-				Values:     map[string]string{string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): strconv.FormatInt(mg.reclaimMemoryLimit.Load(), 10)},
+				Values: map[string]string{
+					string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): strconv.FormatInt(memoryMax, 10),
+					string(memoryadvisor.ControlKnobKeyMemoryHigh):         strconv.FormatInt(memoryHigh, 10),
+				},
 			},
 		},
 	}
@@ -127,9 +138,17 @@ func (mg *memoryGuard) GetAdvices() types.InternalMemoryCalculationResult {
 				continue
 			}
 
+			numaMemoryMax := numaBindingReclaimMemoryLimit[numaID]
+			numaMemoryHigh := int64(float64(numaMemoryMax) * memoryHighScaleFactor)
+			if numaMemoryMax == reclaimMemoryUnlimited {
+				numaMemoryHigh = reclaimMemoryUnlimited
+			}
 			result.ExtraEntries = append(result.ExtraEntries, types.ExtraMemoryAdvices{
 				CgroupPath: cgroupPath,
-				Values:     map[string]string{string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): strconv.FormatInt(numaBindingReclaimMemoryLimit[numaID], 10)},
+				Values: map[string]string{
+					string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): strconv.FormatInt(numaMemoryMax, 10),
+					string(memoryadvisor.ControlKnobKeyMemoryHigh):         strconv.FormatInt(numaMemoryHigh, 10),
+				},
 			})
 		}
 	}

pkg/util/cgroup/common/types.go

@@ -73,7 +73,9 @@ type MemoryData struct {
 	// MinInBytes for memory.min
 	// cgroup memory that can never be reclaimed by kswapd.
 	MinInBytes int64
-	WmarkRatio int32
+	// HighInBytes for memory.high
+	HighInBytes int64
+	WmarkRatio  int32
 	// SwapMaxInBytes < 0 means disable cgroup-level swap
 	SwapMaxInBytes int64
 }

pkg/util/cgroup/manager/v2/fs_linux.go

@@ -73,6 +73,18 @@ func (m *manager) ApplyMemory(absCgroupPath string, data *common.MemoryData) err
 		}
 	}
 
+	if data.HighInBytes != 0 {
+		memoryHighValue := "max"
+		if data.HighInBytes > 0 {
+			memoryHighValue = numToStr(data.HighInBytes)
+		}
+		if err, applied, oldData := common.InstrumentedWriteFileIfChange(absCgroupPath, "memory.high", memoryHighValue); err != nil {
+			return err
+		} else if applied {
+			klog.Infof("[CgroupV2] apply memory high successfully, cgroupPath: %s, data: %v, old data: %v\n", absCgroupPath, memoryHighValue, oldData)
+		}
+	}
+
 	if data.WmarkRatio != 0 {
 		newRatio := fmt.Sprintf("%d", data.WmarkRatio)
 		if err, applied, oldData := common.InstrumentedWriteFileIfChange(absCgroupPath, "memory.wmark_ratio", newRatio); err != nil {