Skip to content

Commit f82de4e

Browse files
lihonghao314lihonghao314
committed
feat(advisor): support numa reclaim reserve
1 parent d77bd6f commit f82de4e

File tree

6 files changed

+95
-23
lines changed

6 files changed

+95
-23
lines changed

‎cmd/katalyst-agent/app/options/dynamic/adminqos/reclaimedresource/reclaimedresource_base.go‎

Lines changed: 23 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,14 +29,15 @@ import (
2929
)
3030

3131
type ReclaimedResourceOptions struct {
32-
EnableReclaim bool
33-
DisableReclaimSharePools []string
34-
ReservedResourceForReport general.ResourceList
35-
MinReclaimedResourceForReport general.ResourceList
36-
MinIgnoredReclaimedResourceForReport general.ResourceList
37-
ReservedResourceForAllocate general.ResourceList
38-
ReservedResourceForReclaimedCores general.ResourceList
39-
32+
EnableReclaim bool
33+
DisableReclaimSharePools []string
34+
ReservedResourceForReport general.ResourceList
35+
MinReclaimedResourceForReport general.ResourceList
36+
MinIgnoredReclaimedResourceForReport general.ResourceList
37+
ReservedResourceForAllocate general.ResourceList
38+
ReservedResourceForReclaimedCores general.ResourceList
39+
NumaMinReservedResourceRatioForAllocate general.ResourceList
40+
NumaMinReservedResourceForAllocate general.ResourceList
4041
*cpuheadroom.CPUHeadroomOptions
4142
*memoryheadroom.MemoryHeadroomOptions
4243
}
@@ -64,6 +65,14 @@ func NewReclaimedResourceOptions() *ReclaimedResourceOptions {
6465
v1.ResourceCPU: resource.MustParse("4"),
6566
v1.ResourceMemory: resource.MustParse("0"),
6667
},
68+
NumaMinReservedResourceRatioForAllocate: map[v1.ResourceName]resource.Quantity{
69+
v1.ResourceCPU: resource.MustParse("0"),
70+
v1.ResourceMemory: resource.MustParse("0"),
71+
},
72+
NumaMinReservedResourceForAllocate: map[v1.ResourceName]resource.Quantity{
73+
v1.ResourceCPU: resource.MustParse("2"),
74+
v1.ResourceMemory: resource.MustParse("0"),
75+
},
6776
CPUHeadroomOptions: cpuheadroom.NewCPUHeadroomOptions(),
6877
MemoryHeadroomOptions: memoryheadroom.NewMemoryHeadroomOptions(),
6978
}
@@ -87,6 +96,10 @@ func (o *ReclaimedResourceOptions) AddFlags(fss *cliflag.NamedFlagSets) {
8796
"reserved reclaimed resource actually not allocate to reclaimed resource")
8897
fs.Var(&o.ReservedResourceForReclaimedCores, "reserved-resource-for-reclaimed-cores",
8998
"reserved resources for reclaimed_cores pods")
99+
fs.Var(&o.NumaMinReservedResourceRatioForAllocate, "numa-min-reserved-resource-ratio-for-reclaimed-cores",
100+
"min NUMA level reserved resources ratio for reclaimed_cores pods")
101+
fs.Var(&o.NumaMinReservedResourceForAllocate, "numa-min-reserved-resource-for-reclaimed-cores",
102+
"min NUMA level reserved resources for reclaimed_cores pods")
90103

91104
o.CPUHeadroomOptions.AddFlags(fss)
92105
o.MemoryHeadroomOptions.AddFlags(fss)
@@ -102,6 +115,8 @@ func (o *ReclaimedResourceOptions) ApplyTo(c *reclaimedresource.ReclaimedResourc
102115
c.MinIgnoredReclaimedResourceForReport = v1.ResourceList(o.MinIgnoredReclaimedResourceForReport)
103116
c.ReservedResourceForAllocate = v1.ResourceList(o.ReservedResourceForAllocate)
104117
c.MinReclaimedResourceForAllocate = v1.ResourceList(o.ReservedResourceForReclaimedCores)
118+
c.NumaMinReclaimedResourceRatioForAllocate = v1.ResourceList(o.NumaMinReservedResourceRatioForAllocate)
119+
c.NumaMinReclaimedResourceForAllocate = v1.ResourceList(o.NumaMinReservedResourceForAllocate)
105120

106121
errList = append(errList, o.CPUHeadroomOptions.ApplyTo(c.CPUHeadroomConfiguration))
107122
errList = append(errList, o.MemoryHeadroomOptions.ApplyTo(c.MemoryHeadroomConfiguration))

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ require (
1919
github.com/google/uuid v1.3.0
2020
github.com/h2non/gock v1.2.0
2121
github.com/klauspost/cpuid/v2 v2.2.6
22-
github.com/kubewharf/katalyst-api v0.5.7-0.20251107024039-d70663b02397
22+
github.com/kubewharf/katalyst-api v0.5.7-0.20251112070927-442dee3bec62
2323
github.com/moby/sys/mountinfo v0.6.2
2424
github.com/montanaflynn/stats v0.7.1
2525
github.com/opencontainers/runc v1.1.6

‎go.sum‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -573,8 +573,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
573573
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
574574
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
575575
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
576-
github.com/kubewharf/katalyst-api v0.5.7-0.20251107024039-d70663b02397 h1:34SMLbHOuwR1JFitzlFOqDnmTEMYZXDRaSXKoKbz/mk=
577-
github.com/kubewharf/katalyst-api v0.5.7-0.20251107024039-d70663b02397/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
576+
github.com/kubewharf/katalyst-api v0.5.7-0.20251112070927-442dee3bec62 h1:D3FZBfIIiZV7e7aQBmTiMPTrc+mlYx+Lreq8wcpTwK8=
577+
github.com/kubewharf/katalyst-api v0.5.7-0.20251112070927-442dee3bec62/go.mod h1:Y2IeIorxQamF2a3oa0+URztl5QCSty6Jj3zD83R8J9k=
578578
github.com/kubewharf/kubelet v1.24.6-kubewharf.9 h1:jOTYZt7h/J7I8xQMKMUcJjKf5UFBv37jHWvNp5VRFGc=
579579
github.com/kubewharf/kubelet v1.24.6-kubewharf.9/go.mod h1:MxbSZUx3wXztFneeelwWWlX7NAAStJ6expqq7gY2J3c=
580580
github.com/kyoh86/exportloopref v0.1.7/go.mod h1:h1rDl2Kdj97+Kwh4gdz3ujE7XHmH51Q0lUiZ1z4NLj8=

‎pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_helper.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,10 @@ package cpu
1818

1919
import (
2020
"fmt"
21+
"math"
2122

2223
v1 "k8s.io/api/core/v1"
24+
"k8s.io/apimachinery/pkg/api/resource"
2325
"k8s.io/apimachinery/pkg/util/sets"
2426
"k8s.io/klog/v2"
2527
"k8s.io/kubelet/pkg/apis/resourceplugin/v1alpha1"
@@ -186,6 +188,13 @@ func (cra *cpuResourceAdvisor) updateNumasAvailableResource() {
186188
}
187189

188190
func (cra *cpuResourceAdvisor) updateReservedForReclaim() {
191+
numaReservedRatio := cra.conf.GetDynamicConfiguration().NumaMinReclaimedResourceRatioForAllocate[v1.ResourceCPU]
192+
if numaReservedRatio.Value() != 0 {
193+
numaReserved := cra.conf.GetDynamicConfiguration().NumaMinReclaimedResourceForAllocate[v1.ResourceCPU]
194+
cra.updateReservedForReclaimByNuma(numaReservedRatio, numaReserved)
195+
return
196+
}
197+
189198
coreNumReservedForReclaim := cra.conf.GetDynamicConfiguration().MinReclaimedResourceForAllocate[v1.ResourceCPU]
190199
if coreNumReservedForReclaim.Value() > int64(cra.metaServer.NumCPUs) {
191200
coreNumReservedForReclaim.Set(int64(cra.metaServer.NumCPUs))
@@ -196,6 +205,21 @@ func (cra *cpuResourceAdvisor) updateReservedForReclaim() {
196205
coreNumReservedForReclaim.Set(int64(cra.metaServer.NumNUMANodes))
197206
}
198207
cra.reservedForReclaim = machine.GetCoreNumReservedForReclaim(int(coreNumReservedForReclaim.Value()), cra.metaServer.NumNUMANodes)
208+
general.Infof("reservedForReclaim: %v, coreNumReservedForReclaim %v", cra.reservedForReclaim, coreNumReservedForReclaim.Value())
209+
}
210+
211+
func (cra *cpuResourceAdvisor) updateReservedForReclaimByNuma(numaReservedRatio resource.Quantity,
212+
numaReserved resource.Quantity,
213+
) {
214+
reservedForReclaim := make(map[int]int)
215+
for id := 0; id < cra.metaServer.NumNUMANodes; id++ {
216+
size := cra.metaServer.NUMAToCPUs.CPUSizeInNUMAs(id)
217+
reserved := math.Ceil(numaReservedRatio.AsApproximateFloat64() * float64(size))
218+
reservedForReclaim[id] = int(math.Max(numaReserved.AsApproximateFloat64(), reserved))
219+
}
220+
cra.reservedForReclaim = reservedForReclaim
221+
general.Infof("reservedForReclaim: %v, numaReservedRatio %v, numaReserved %v",
222+
reservedForReclaim, numaReservedRatio.AsApproximateFloat64(), numaReserved.AsApproximateFloat64())
199223
}
200224

201225
func (cra *cpuResourceAdvisor) getNumasReservedForAllocate(numas machine.CPUSet) float64 {

‎pkg/agent/sysadvisor/plugin/qosaware/resource/cpu/advisor_helper_test.go‎

Lines changed: 25 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,11 @@ import (
3434
func Test_cpuResourceAdvisor_updateReservedForReclaim(t *testing.T) {
3535
t.Parallel()
3636
type fields struct {
37-
numaNum int
38-
socketNum int
39-
numCPUs int
40-
minReclaimedResourceForAllocate v1.ResourceList
37+
numaNum int
38+
socketNum int
39+
numCPUs int
40+
minReclaimedResourceForAllocate v1.ResourceList
41+
numaMinReclaimedResourceRatioForAllocate v1.ResourceList
4142
}
4243
tests := []struct {
4344
name string
@@ -95,6 +96,24 @@ func Test_cpuResourceAdvisor_updateReservedForReclaim(t *testing.T) {
9596
7: 1,
9697
},
9798
},
99+
{
100+
name: "reserved with numa size ratio",
101+
fields: fields{
102+
numaNum: 2,
103+
socketNum: 1,
104+
numCPUs: 64,
105+
minReclaimedResourceForAllocate: v1.ResourceList{
106+
v1.ResourceCPU: resource.MustParse("4"),
107+
},
108+
numaMinReclaimedResourceRatioForAllocate: v1.ResourceList{
109+
v1.ResourceCPU: resource.MustParse("0.05"),
110+
},
111+
},
112+
wantReservedForReclaim: map[int]int{
113+
0: 2,
114+
1: 2,
115+
},
116+
},
98117
}
99118
for _, tt := range tests {
100119
tt := tt
@@ -110,8 +129,9 @@ func Test_cpuResourceAdvisor_updateReservedForReclaim(t *testing.T) {
110129
defer func() { _ = os.RemoveAll(sfDir) }()
111130

112131
conf := generateTestConfiguration(t, ckDir, sfDir)
113-
132+
conf.GetDynamicConfiguration().EnableStrategyGroup = true
114133
conf.GetDynamicConfiguration().MinReclaimedResourceForAllocate = tt.fields.minReclaimedResourceForAllocate
134+
conf.GetDynamicConfiguration().NumaMinReclaimedResourceRatioForAllocate = tt.fields.numaMinReclaimedResourceRatioForAllocate
115135

116136
cpuTopology, err := machine.GenerateDummyCPUTopology(tt.fields.numCPUs, tt.fields.socketNum, tt.fields.numaNum)
117137
assert.NoError(t, err)

‎pkg/config/agent/dynamic/adminqos/reclaimedresource/reclaimedresource_base.go‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -25,13 +25,15 @@ import (
2525
)
2626

2727
type ReclaimedResourceConfiguration struct {
28-
EnableReclaim bool
29-
DisableReclaimSharePools []string
30-
ReservedResourceForReport v1.ResourceList
31-
MinReclaimedResourceForReport v1.ResourceList
32-
MinIgnoredReclaimedResourceForReport v1.ResourceList
33-
ReservedResourceForAllocate v1.ResourceList
34-
MinReclaimedResourceForAllocate v1.ResourceList
28+
EnableReclaim bool
29+
DisableReclaimSharePools []string
30+
ReservedResourceForReport v1.ResourceList
31+
MinReclaimedResourceForReport v1.ResourceList
32+
MinIgnoredReclaimedResourceForReport v1.ResourceList
33+
ReservedResourceForAllocate v1.ResourceList
34+
MinReclaimedResourceForAllocate v1.ResourceList
35+
NumaMinReclaimedResourceRatioForAllocate v1.ResourceList
36+
NumaMinReclaimedResourceForAllocate v1.ResourceList
3537

3638
*cpuheadroom.CPUHeadroomConfiguration
3739
*memoryheadroom.MemoryHeadroomConfiguration
@@ -84,6 +86,17 @@ func (c *ReclaimedResourceConfiguration) ApplyConfiguration(conf *crd.DynamicCon
8486
c.MinReclaimedResourceForAllocate[resourceName] = value
8587
}
8688
}
89+
90+
if config.NumaMinReclaimedResourceRatioForAllocate != nil {
91+
for resourceName, value := range *config.NumaMinReclaimedResourceRatioForAllocate {
92+
c.NumaMinReclaimedResourceRatioForAllocate[resourceName] = value
93+
}
94+
}
95+
if config.NumaMinReclaimedResourceForAllocate != nil {
96+
for resourceName, value := range *config.NumaMinReclaimedResourceForAllocate {
97+
c.NumaMinReclaimedResourceForAllocate[resourceName] = value
98+
}
99+
}
87100
}
88101

89102
c.CPUHeadroomConfiguration.ApplyConfiguration(conf)

0 commit comments

Comments
 (0)