diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/memoryadvisor/types.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/memoryadvisor/types.go index c620614c4e..4ec798a3d5 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/memoryadvisor/types.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/memoryadvisor/types.go @@ -28,6 +28,7 @@ const ( ControlKnowKeyMemoryOffloading MemoryControlKnobName = "memory_offloading" ControlKnowKeyDyingMemcgReclaim MemoryControlKnobName = "dying_memcg_reclaim" ControlKnobKeyMemoryNUMAHeadroom MemoryControlKnobName = "memory_numa_headroom" + ControlKnobKeyMemoryHigh MemoryControlKnobName = "memory_high" ) const ( diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go index 0840cf9125..7f565b68ce 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy.go @@ -273,6 +273,8 @@ func NewDynamicPolicy(agentCtx *agent.GenericContext, conf *config.Configuration memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyMemoryLimitInBytes, memoryadvisor.ControlKnobHandlerWithChecker(policyImplement.handleAdvisorMemoryLimitInBytes)) + memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyMemoryHigh, + memoryadvisor.ControlKnobHandlerWithChecker(policyImplement.handleAdvisorMemoryHigh)) memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyCPUSetMems, memoryadvisor.ControlKnobHandlerWithChecker(policyImplement.handleAdvisorCPUSetMems)) memoryadvisor.RegisterControlKnobHandler(memoryadvisor.ControlKnobKeyDropCache, diff --git a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go index 021f017e44..570382804e 100644 --- a/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go +++ b/pkg/agent/qrm-plugins/memory/dynamicpolicy/policy_advisor_handler.go @@ -26,6 +26,7 @@ import ( "strconv" "time" + "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/samber/lo" "google.golang.org/grpc" "google.golang.org/grpc/metadata" @@ -392,6 +393,43 @@ func (p *DynamicPolicy) handleAdvisorResp(advisorResp *advisorsvc.ListAndWatchRe return nil } +func (p *DynamicPolicy) handleAdvisorMemoryHigh( + _ *config.Configuration, + _ interface{}, + _ *dynamicconfig.DynamicAgentConfiguration, + emitter metrics.MetricEmitter, + metaServer *metaserver.MetaServer, + entryName, subEntryName string, + calculationInfo *advisorsvc.CalculationInfo, podResourceEntries state.PodResourceEntries, +) error { + memoryHighStr := calculationInfo.CalculationResult.Values[string(memoryadvisor.ControlKnobKeyMemoryHigh)] + memoryHigh, err := strconv.ParseInt(memoryHighStr, 10, 64) + if err != nil { + return fmt.Errorf("parse %s: %s failed with error: %v", memoryadvisor.ControlKnobKeyMemoryHigh, memoryHighStr, err) + } + + if !cgroups.IsCgroup2UnifiedMode() { + general.Infof("memory.high is not supported in cgroupv1 mode") + return nil + } + + if calculationInfo.CgroupPath != "" { + if err = cgroupmgr.ApplyMemoryWithRelativePath(calculationInfo.CgroupPath, &common.MemoryData{ + HighInBytes: memoryHigh, + }); err != nil { + return fmt.Errorf("apply memory.high failed with error: %v", err) + } + + _ = emitter.StoreInt64(util.MetricNameMemoryHandleAdvisorMemoryHigh, memoryHigh, + metrics.MetricTypeNameRaw, metrics.ConvertMapToTags(map[string]string{ + "cgroupPath": calculationInfo.CgroupPath, + })...) + return nil + } + + return nil +} + func (p *DynamicPolicy) handleAdvisorMemoryLimitInBytes( _ *config.Configuration, _ interface{}, diff --git a/pkg/agent/qrm-plugins/util/consts.go b/pkg/agent/qrm-plugins/util/consts.go index 4b49145699..41c25842f2 100644 --- a/pkg/agent/qrm-plugins/util/consts.go +++ b/pkg/agent/qrm-plugins/util/consts.go @@ -51,6 +51,7 @@ const ( MetricNameMemoryHandleAdvisorContainerEntryFailed = "memory_handle_advisor_container_entry_failed" MetricNameMemoryHandleAdvisorExtraEntryFailed = "memory_handle_advisor_extra_entry_failed" MetricNameMemoryHandleAdvisorMemoryLimit = "memory_handle_advisor_memory_limit" + MetricNameMemoryHandleAdvisorMemoryHigh = "memory_handle_advisor_memory_high" MetricNameMemoryHandleAdvisorDropCache = "memory_handle_advisor_drop_cache" MetricNameMemoryHandleAdvisorCPUSetMems = "memory_handle_advisor_cpuset_mems" MetricNameMemoryHandlerAdvisorMemoryOffload = "memory_handler_advisor_memory_offloading" diff --git a/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/plugin/memory_guard.go b/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/plugin/memory_guard.go index 99c9ff5d1f..6935b329da 100644 --- a/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/plugin/memory_guard.go +++ b/pkg/agent/sysadvisor/plugin/qosaware/resource/memory/plugin/memory_guard.go @@ -44,6 +44,9 @@ const ( reclaimMemoryUnlimited = -1 defaultProcZoneInfoFile = "/proc/zoneinfo" + + // Factor for calculating memory.high based on memory.max, available for cgroup v2 only + memoryHighScaleFactor = 0.95 ) type memoryGuard struct { @@ -110,11 +113,19 @@ func (mg *memoryGuard) GetAdvices() types.InternalMemoryCalculationResult { general.Errorf("failed to get last reconcile result") return types.InternalMemoryCalculationResult{} } + memoryMax := mg.reclaimMemoryLimit.Load() + memoryHigh := int64(float64(memoryMax) * memoryHighScaleFactor) + if memoryMax == reclaimMemoryUnlimited { + memoryHigh = reclaimMemoryUnlimited + } result := types.InternalMemoryCalculationResult{ ExtraEntries: []types.ExtraMemoryAdvices{ { CgroupPath: mg.reclaimRelativeRootCgroupPath, - Values: map[string]string{string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): strconv.FormatInt(mg.reclaimMemoryLimit.Load(), 10)}, + Values: map[string]string{ + string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): strconv.FormatInt(memoryMax, 10), + string(memoryadvisor.ControlKnobKeyMemoryHigh): strconv.FormatInt(memoryHigh, 10), + }, }, }, } @@ -127,9 +138,17 @@ func (mg *memoryGuard) GetAdvices() types.InternalMemoryCalculationResult { continue } + numaMemoryMax := numaBindingReclaimMemoryLimit[numaID] + numaMemoryHigh := int64(float64(numaMemoryMax) * memoryHighScaleFactor) + if numaMemoryMax == reclaimMemoryUnlimited { + numaMemoryHigh = reclaimMemoryUnlimited + } result.ExtraEntries = append(result.ExtraEntries, types.ExtraMemoryAdvices{ CgroupPath: cgroupPath, - Values: map[string]string{string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): strconv.FormatInt(numaBindingReclaimMemoryLimit[numaID], 10)}, + Values: map[string]string{ + string(memoryadvisor.ControlKnobKeyMemoryLimitInBytes): strconv.FormatInt(numaMemoryMax, 10), + string(memoryadvisor.ControlKnobKeyMemoryHigh): strconv.FormatInt(numaMemoryHigh, 10), + }, }) } } diff --git a/pkg/util/cgroup/common/types.go b/pkg/util/cgroup/common/types.go index d92f4cef10..a681afd55e 100644 --- a/pkg/util/cgroup/common/types.go +++ b/pkg/util/cgroup/common/types.go @@ -73,7 +73,9 @@ type MemoryData struct { // MinInBytes for memory.min // cgroup memory that can never be reclaimed by kswapd. MinInBytes int64 - WmarkRatio int32 + // HighInBytes for memory.high + HighInBytes int64 + WmarkRatio int32 // SwapMaxInBytes < 0 means disable cgroup-level swap SwapMaxInBytes int64 } diff --git a/pkg/util/cgroup/manager/v2/fs_linux.go b/pkg/util/cgroup/manager/v2/fs_linux.go index 0ac25b2831..f614a9f97a 100644 --- a/pkg/util/cgroup/manager/v2/fs_linux.go +++ b/pkg/util/cgroup/manager/v2/fs_linux.go @@ -73,6 +73,18 @@ func (m *manager) ApplyMemory(absCgroupPath string, data *common.MemoryData) err } } + if data.HighInBytes != 0 { + memoryHighValue := "max" + if data.HighInBytes > 0 { + memoryHighValue = numToStr(data.HighInBytes) + } + if err, applied, oldData := common.InstrumentedWriteFileIfChange(absCgroupPath, "memory.high", memoryHighValue); err != nil { + return err + } else if applied { + klog.Infof("[CgroupV2] apply memory high successfully, cgroupPath: %s, data: %v, old data: %v\n", absCgroupPath, memoryHighValue, oldData) + } + } + if data.WmarkRatio != 0 { newRatio := fmt.Sprintf("%d", data.WmarkRatio) if err, applied, oldData := common.InstrumentedWriteFileIfChange(absCgroupPath, "memory.wmark_ratio", newRatio); err != nil {