Fix 504 timeout: Add K8s client caching + QPS increase + benchmarks

Signed-off-by: shovan-mondal <shovanmondal2004@gmail.com>

Author: shovan-mondal

chaoscenter/subscriber/pkg/k8s/client.go

@@ -1,6 +1,9 @@
 package k8s
 
 import (
+	"sync"
+	"time"
+
 	wfclientset "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned"
 	v1alpha12 "github.com/argoproj/argo-workflows/v3/pkg/client/clientset/versioned/typed/workflow/v1alpha1"
 	"k8s.io/client-go/discovery"
@@ -10,24 +13,55 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 )
 
-var KubeConfig *string
+var (
+	KubeConfig *string
+
+	// Singleton clientset to avoid creating new clients per request
+	clientsetOnce     sync.Once
+	clientsetInstance *kubernetes.Clientset
+	clientsetErr      error
+)
 
 // getKubeConfig setup the config for access cluster resource
 func (k8s *k8sSubscriber) GetKubeConfig() (*rest.Config, error) {
+	var config *rest.Config
+	var err error
+
 	// Use in-cluster config if kubeconfig path is not specified
 	if *KubeConfig == "" {
-		return rest.InClusterConfig()
+		config, err = rest.InClusterConfig()
+	} else {
+		config, err = clientcmd.BuildConfigFromFlags("", *KubeConfig)
 	}
-	return clientcmd.BuildConfigFromFlags("", *KubeConfig)
-}
 
-func (k8s *k8sSubscriber) GetGenericK8sClient() (*kubernetes.Clientset, error) {
-	config, err := k8s.GetKubeConfig()
 	if err != nil {
 		return nil, err
 	}
 
-	return kubernetes.NewForConfig(config)
+	// Default QPS=5 and Burst=10 are too low for clusters with 100+ namespaces
+	config.QPS = 50.0
+	config.Burst = 100
+	config.Timeout = 30 * time.Second
+
+	return config, nil
+}
+
+func (k8s *k8sSubscriber) GetGenericK8sClient() (*kubernetes.Clientset, error) {
+	// This eliminates TCP handshake and TLS negotiation overhead
+	clientsetOnce.Do(func() {
+		config, err := k8s.GetKubeConfig()
+		if err != nil {
+			clientsetErr = err
+			return
+		}
+		clientsetInstance, clientsetErr = kubernetes.NewForConfig(config)
+	})
+
+	if clientsetErr != nil {
+		return nil, clientsetErr
+	}
+
+	return clientsetInstance, nil
 }
 
 // This function returns dynamic client and discovery client

chaoscenter/subscriber/pkg/k8s/objects.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 	"strings"
+	"time"
 
 	"github.com/sirupsen/logrus"
 
@@ -15,7 +16,6 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/client-go/dynamic"
-	"k8s.io/client-go/kubernetes"
 )
 
 var (
@@ -35,30 +35,41 @@ func (k8s *k8sSubscriber) GetKubernetesNamespaces(request types.KubeNamespaceReq
 		}
 		namespaceData = append(namespaceData, KubeNamespace)
 	} else {
-		// In case of cluster scope, get all the namespaces
-		conf, err := k8s.GetKubeConfig()
-		if err != nil {
-			return nil, err
-		}
-		clientset, err := kubernetes.NewForConfig(conf)
+		// Cached clientset instead of creating new one
+		clientset, err := k8s.GetGenericK8sClient()
 		if err != nil {
 			return nil, err
 		}
 
-		namespace, err := clientset.CoreV1().Namespaces().List(context.TODO(), metav1.ListOptions{})
-		if err != nil {
-			return nil, err
+		// Add context timeout to prevent indefinite hangs
+		ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+		defer cancel()
+
+		// Add pagination support for clusters with many namespaces
+		listOpts := metav1.ListOptions{
+			Limit: 500, // Fetch in batches of 500
 		}
-		if len(namespace.Items) > 0 {
-			for _, namespace := range namespace.Items {
 
-				KubeNamespace := &types.KubeNamespace{
-					Name: namespace.GetName(),
-				}
+		for {
+			namespaceList, err := clientset.CoreV1().Namespaces().List(ctx, listOpts)
+			if err != nil {
+				return nil, err
+			}
+
+			for _, ns := range namespaceList.Items {
+				namespaceData = append(namespaceData, &types.KubeNamespace{
+					Name: ns.GetName(),
+				})
+			}
 
-				namespaceData = append(namespaceData, KubeNamespace)
+			// Check if there are more results to fetch
+			if namespaceList.Continue == "" {
+				break
 			}
-		} else {
+			listOpts.Continue = namespaceList.Continue
+		}
+
+		if len(namespaceData) == 0 {
 			return nil, errors.New("No namespace available")
 		}
 	}