Skip to content

Commit c5e808b

Browse files
RamLaviRamLavi
authored
Optimize init map (#555)
* pool-manager/pods: Optimize InitMap by pre-filtering managed namespaces Currently kubemacpool scans all pods cluster-wide and performs expensive IsPodManaged() checks for each pod during startup. This causes slow initialization in large clusters with many unmanaged namespaces. Pre-compute managed namespaces once per webhook type and only scan pods from those namespaces, eliminating redundant API calls and reducing startup time. Signed-off-by: Ram Lavi <ralavi@redhat.com> * pool-manager/vms: Optimize InitMap by pre-filtering managed namespaces Currently kubemacpool scans all VMs cluster-wide and performs expensive IsVMManaged() checks for each VM during startup. This causes slow initialization in large clusters with many unmanaged namespaces. Pre-compute managed namespaces once per webhook type and only scan VMs from those namespaces, eliminating redundant API calls and reducing startup time. Signed-off-by: Ram Lavi <ralavi@redhat.com> * config: Increase readiness wait to 180s To anticipate big clusters with 10K VMs, increasing the readiness probe settle time to 3m. Signed-off-by: Ram Lavi <ralavi@redhat.com> --------- Signed-off-by: Ram Lavi <ralavi@redhat.com>
1 parent 910acf7 commit c5e808b

File tree

6 files changed

+82
-37
lines changed

6 files changed

+82
-37
lines changed

‎config/default/manager/manager.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -131,7 +131,7 @@ spec:
131131
port: webhook-server
132132
scheme: HTTPS
133133
initialDelaySeconds: 10
134-
periodSeconds: 10
134+
periodSeconds: 180
135135
livenessProbe:
136136
httpGet:
137137
httpHeaders:

‎config/release/kubemacpool.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -307,7 +307,7 @@ spec:
307307
port: webhook-server
308308
scheme: HTTPS
309309
initialDelaySeconds: 10
310-
periodSeconds: 10
310+
periodSeconds: 180
311311
resources:
312312
requests:
313313
cpu: 100m

‎config/test/kubemacpool.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,7 +308,7 @@ spec:
308308
port: webhook-server
309309
scheme: HTTPS
310310
initialDelaySeconds: 10
311-
periodSeconds: 10
311+
periodSeconds: 180
312312
resources:
313313
requests:
314314
cpu: 100m

‎pkg/pool-manager/pod_pool.go‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ import (
2626
"github.com/pkg/errors"
2727
multus "gopkg.in/k8snetworkplumbingwg/multus-cni.v3/pkg/types"
2828
corev1 "k8s.io/api/core/v1"
29-
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3029
"sigs.k8s.io/controller-runtime/pkg/client"
3130

3231
networkv1 "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/apis/k8s.cni.cncf.io/v1"
@@ -194,14 +193,36 @@ func (p *PoolManager) allocatePodFromPool(network *multus.NetworkSelectionElemen
194193
return macAddr.String(), nil
195194
}
196195

197-
// paginatePodsWithLimit performs a pods list request with pagination, to limit the amount of pods received at a time
198-
// and prevent exceeding the memory limit.
199-
func (p *PoolManager) paginatePodsWithLimit(limit int64, f func(pods *corev1.PodList) error) error {
196+
// paginatePodsInManagedNamespaces performs pod list requests with pagination, but only for managed namespaces
197+
func (p *PoolManager) paginatePodsInManagedNamespaces(limit int64, f func(pods *corev1.PodList) error) error {
198+
managedNamespaces, err := p.getManagedNamespaces(podsWebhookName)
199+
if err != nil {
200+
return errors.Wrap(err, "failed to get managed namespaces for pods")
201+
}
202+
203+
if len(managedNamespaces) == 0 {
204+
log.Info("no managed namespaces found, skipping pod initialization")
205+
return nil
206+
}
207+
208+
for _, namespace := range managedNamespaces {
209+
log.V(1).Info("processing pods in managed namespace", "namespace", namespace)
210+
err := p.paginatePodsInNamespace(namespace, limit, f)
211+
if err != nil {
212+
return errors.Wrapf(err, "failed to process pods in namespace %s", namespace)
213+
}
214+
}
215+
216+
return nil
217+
}
218+
219+
// paginatePodsInNamespace performs pods list request with pagination for a specific namespace
220+
func (p *PoolManager) paginatePodsInNamespace(namespace string, limit int64, f func(pods *corev1.PodList) error) error {
200221
continueFlag := ""
201222
for {
202223
pods := corev1.PodList{}
203224
err := p.kubeClient.List(context.TODO(), &pods, &client.ListOptions{
204-
Namespace: metav1.NamespaceAll,
225+
Namespace: namespace,
205226
Limit: limit,
206227
Continue: continueFlag,
207228
})
@@ -215,7 +236,7 @@ func (p *PoolManager) paginatePodsWithLimit(limit int64, f func(pods *corev1.Pod
215236
}
216237

217238
continueFlag = pods.GetContinue()
218-
log.V(1).Info("limit Pod list", "pods len", len(pods.Items), "remaining", pods.GetRemainingItemCount(), "continue", continueFlag)
239+
log.V(1).Info("limit Pod list in namespace", "namespace", namespace, "pods len", len(pods.Items), "remaining", pods.GetRemainingItemCount(), "continue", continueFlag)
219240
if continueFlag == "" {
220241
break
221242
}
@@ -225,7 +246,7 @@ func (p *PoolManager) paginatePodsWithLimit(limit int64, f func(pods *corev1.Pod
225246

226247
func (p *PoolManager) initPodMap() error {
227248
log.V(1).Info("start InitMaps to reserve existing mac addresses before allocation new ones")
228-
err := p.paginatePodsWithLimit(100, func(pods *corev1.PodList) error {
249+
err := p.paginatePodsInManagedNamespaces(100, func(pods *corev1.PodList) error {
229250
for _, pod := range pods.Items {
230251
log.V(1).Info("InitMaps for pod", "podName", pod.Name, "podNamespace", pod.Namespace)
231252
if pod.Annotations == nil {
@@ -237,15 +258,6 @@ func (p *PoolManager) initPodMap() error {
237258
continue
238259
}
239260

240-
instanceManaged, err := p.IsPodManaged(pod.GetNamespace())
241-
if err != nil {
242-
continue
243-
}
244-
245-
if !instanceManaged {
246-
continue
247-
}
248-
249261
networks, err := parsePodNetworkAnnotation(networkValue, pod.Namespace)
250262
if err != nil {
251263
continue
@@ -282,7 +294,7 @@ func (p *PoolManager) initPodMap() error {
282294
return nil
283295
})
284296
if err != nil {
285-
return errors.Wrap(err, "failed iterating over all cluster pods")
297+
return errors.Wrap(err, "failed iterating over pods in managed namespaces")
286298
}
287299

288300
return nil

‎pkg/pool-manager/pool.go‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -142,6 +142,27 @@ func (p *PoolManager) Start() error {
142142
return nil
143143
}
144144

145+
// getManagedNamespaces pre-computes which namespaces are managed by kubemacpool for a specific webhook
146+
func (p *PoolManager) getManagedNamespaces(webhookName string) ([]string, error) {
147+
log.V(1).Info("computing managed namespaces for initialization", "webhookName", webhookName)
148+
149+
namespaces := &v1.NamespaceList{}
150+
err := p.kubeClient.List(context.TODO(), namespaces)
151+
if err != nil {
152+
return nil, errors.Wrapf(err, "failed to list namespaces for webhook %s", webhookName)
153+
}
154+
155+
var managedNamespaces []string
156+
for _, ns := range namespaces.Items {
157+
if managed, err := p.IsNamespaceManaged(ns.Name, webhookName); err == nil && managed {
158+
managedNamespaces = append(managedNamespaces, ns.Name)
159+
}
160+
}
161+
162+
log.Info("computed managed namespaces", "webhookName", webhookName, "count", len(managedNamespaces), "namespaces", managedNamespaces)
163+
return managedNamespaces, nil
164+
}
165+
145166
func (p *PoolManager) InitMaps() error {
146167
err := p.initPodMap()
147168
if err != nil {

‎pkg/pool-manager/virtualmachine_pool.go‎

Lines changed: 29 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -314,15 +314,36 @@ func (p *PoolManager) initMacMapFromCluster(parentLogger logr.Logger) error {
314314
return nil
315315
}
316316

317-
// paginateVmsWithLimit performs a vm list request with pagination, to limit the amount of vms received at a time
318-
// and prevent taking too much memory.
319-
func (p *PoolManager) paginateVmsWithLimit(limit int64, vmsFunc func(pods *kubevirt.VirtualMachineList) error) error {
317+
// paginateVmsInManagedNamespaces performs VM list requests with pagination, but only for managed namespaces
318+
func (p *PoolManager) paginateVmsInManagedNamespaces(limit int64, vmsFunc func(vms *kubevirt.VirtualMachineList) error) error {
319+
managedNamespaces, err := p.getManagedNamespaces(virtualMachnesWebhookName)
320+
if err != nil {
321+
return errors.Wrap(err, "failed to get managed namespaces for VMs")
322+
}
323+
324+
if len(managedNamespaces) == 0 {
325+
log.Info("no managed namespaces found for VMs, skipping VM initialization")
326+
return nil
327+
}
328+
329+
for _, namespace := range managedNamespaces {
330+
log.V(1).Info("processing VMs in managed namespace", "namespace", namespace)
331+
err := p.paginateVmsInNamespace(namespace, limit, vmsFunc)
332+
if err != nil {
333+
return errors.Wrapf(err, "failed to process VMs in namespace %s", namespace)
334+
}
335+
}
336+
337+
return nil
338+
}
339+
340+
// paginateVmsInNamespace performs VM list request with pagination for a specific namespace
341+
func (p *PoolManager) paginateVmsInNamespace(namespace string, limit int64, vmsFunc func(vms *kubevirt.VirtualMachineList) error) error {
320342
continueFlag := ""
321343
for {
322-
// Using a unstructured object.
323344
vms := &kubevirt.VirtualMachineList{}
324345
err := p.kubeClient.List(context.TODO(), vms, &client.ListOptions{
325-
Namespace: metav1.NamespaceAll,
346+
Namespace: namespace,
326347
Limit: limit,
327348
Continue: continueFlag,
328349
})
@@ -336,7 +357,7 @@ func (p *PoolManager) paginateVmsWithLimit(limit int64, vmsFunc func(pods *kubev
336357
}
337358

338359
continueFlag = vms.GetContinue()
339-
log.V(1).Info("limit vms list", "vms len", len(vms.Items), "remaining", vms.GetRemainingItemCount(), "continue", continueFlag)
360+
log.V(1).Info("limit vms list in namespace", "namespace", namespace, "vms len", len(vms.Items), "remaining", vms.GetRemainingItemCount(), "continue", continueFlag)
340361
if continueFlag == "" {
341362
break
342363
}
@@ -347,18 +368,9 @@ func (p *PoolManager) paginateVmsWithLimit(limit int64, vmsFunc func(pods *kubev
347368
// forEachManagedVmInterfaceInClusterRunFunction gets all the macs from all the supported interfaces in all the managed cluster vms, and runs
348369
// a function vmInterfacesFunc on it
349370
func (p *PoolManager) forEachManagedVmInterfaceInClusterRunFunction(vmInterfacesFunc func(vmFullName string, iface kubevirt.Interface, networks map[string]kubevirt.Network) error) error {
350-
err := p.paginateVmsWithLimit(100, func(vms *kubevirt.VirtualMachineList) error {
371+
err := p.paginateVmsInManagedNamespaces(100, func(vms *kubevirt.VirtualMachineList) error {
351372
logger := log.WithName("forEachManagedVmInterfaceInClusterRunFunction")
352373
for _, vm := range vms.Items {
353-
vmNamespace := vm.GetNamespace()
354-
isNamespaceManaged, err := p.IsVirtualMachineManaged(vmNamespace)
355-
if err != nil {
356-
return errors.Wrap(err, fmt.Sprintf("failed to check if namespace %s is managed in current opt-mode", vmNamespace))
357-
}
358-
if !isNamespaceManaged {
359-
logger.V(1).Info("skipping vm in loop iteration, namespace not managed", "vmNamespace", vmNamespace)
360-
continue
361-
}
362374
vmFullName := VmNamespaced(&vm)
363375
vmInterfaces := getVirtualMachineInterfaces(&vm)
364376
vmNetworks := getVirtualMachineNetworks(&vm)
@@ -393,7 +405,7 @@ func (p *PoolManager) forEachManagedVmInterfaceInClusterRunFunction(vmInterfaces
393405
return nil
394406
})
395407
if err != nil {
396-
return errors.Wrap(err, "failed iterating over all cluster vms")
408+
return errors.Wrap(err, "failed iterating over VMs in managed namespaces")
397409
}
398410
return nil
399411
}

0 commit comments

Comments
 (0)