feat: Add an ability to measure clusters with dns only control plane

clusterloader2/cmd/clusterloader.go

@@ -91,6 +91,7 @@ func initClusterFlags() {
 	// TODO(#595): Change the name of the MASTER_IP and MASTER_INTERNAL_IP flags and vars to plural
 	flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterIPs, "masterip", "MASTER_IP", nil /*defaultValue*/, "Hostname/IP of the master node, supports multiple values when separated by commas")
 	flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterInternalIPs, "master-internal-ip", "MASTER_INTERNAL_IP", nil /*defaultValue*/, "Cluster internal/private IP of the master vm, supports multiple values when separated by commas")
+	flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint, "master-endpoint", "MASTER_DNS_ENDPOINT", "", "Endpoint of the master node, exclusive with --masterip and --master-internal-ips")
 	flags.BoolEnvVar(&clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled, "apiserver-pprof-by-client-enabled", "APISERVER_PPROF_BY_CLIENT_ENABLED", true, "Whether apiserver pprof endpoint can be accessed by Kubernetes client.")
 	flags.BoolVar(&clusterLoaderConfig.ClusterConfig.SkipClusterVerification, "skip-cluster-verification", false, "Whether to skip the cluster verification, which expects at least one schedulable node in the cluster")
 
@@ -167,6 +168,23 @@ func completeConfig(m *framework.MultiClientSet) error {
 		clusterLoaderConfig.ClusterConfig.Nodes = nodes
 		klog.V(0).Infof("ClusterConfig.Nodes set to %v", nodes)
 	}
+	if clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint == "" {
+		err := completeIpMasterConfig(m)
+		if err != nil {
+			return err
+		}
+	}
+
+	if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint {
+		clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false
+	}
+	if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 {
+		clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes)
+	}
+	return nil
+}
+
+func completeIpMasterConfig(m *framework.MultiClientSet) error {
 	if clusterLoaderConfig.ClusterConfig.MasterName == "" {
 		masterName, err := util.GetMasterName(m.GetClient())
 		if err == nil {
@@ -194,13 +212,6 @@ func completeConfig(m *framework.MultiClientSet) error {
 			klog.Errorf("Getting master internal ip error: %v", err)
 		}
 	}
-
-	if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint {
-		clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false
-	}
-	if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 {
-		clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes)
-	}
 	return nil
 }
 
@@ -288,6 +299,15 @@ func main() {
 		klog.Exitf("Parsing flags error: %v", errList.String())
 	}
 
+	klog.V(2).Infof("KubeConfigPath: %v", clusterLoaderConfig.ClusterConfig.KubeConfigPath)
+	if clusterLoaderConfig.ClusterConfig.KubeConfigPath != "" {
+		content, err := ioutil.ReadFile(clusterLoaderConfig.ClusterConfig.KubeConfigPath)
+		if err != nil {
+			klog.Errorf("Error reading kubeconfig: %v", err)
+		} else {
+			klog.V(2).Infof("KubeConfig content:\n%s", string(content))
+		}
+	}
 	mclient, err := framework.NewMultiClientSet(clusterLoaderConfig.ClusterConfig.KubeConfigPath, 1)
 	if err != nil {
 		klog.Exitf("Client creation error: %v", err)

clusterloader2/pkg/config/cluster.go

@@ -46,6 +46,7 @@ type ClusterConfig struct {
 	MasterIPs           []string
 	MasterInternalIPs   []string
 	MasterName          string
+	MasterDNSEndpoint   string
 	// Deprecated: use NamespaceConfig.DeleteStaleNamespaces instead.
 	DeleteStaleNamespaces bool
 	// TODO(#1696): Clean up after removing automanagedNamespaces

clusterloader2/pkg/measurement/common/profile.go

@@ -67,6 +67,9 @@ func (p *profileMeasurement) populateProfileConfig(config *measurement.Config) e
 	}
 	p.config.provider = config.ClusterFramework.GetClusterConfig().Provider
 	p.config.hosts = config.ClusterFramework.GetClusterConfig().MasterIPs
+	if masterDnsEnpoint := config.ClusterFramework.GetClusterConfig().MasterDNSEndpoint; masterDnsEnpoint != "" {
+		p.config.hosts = append(p.config.hosts, masterDnsEnpoint)
+	}
 	return nil
 }
 

clusterloader2/pkg/prometheus/manifests/master-ip/master-endpoints.yaml

@@ -1,36 +1,54 @@
 {{$PROMETHEUS_SCRAPE_NODE_EXPORTER := DefaultParam .PROMETHEUS_SCRAPE_NODE_EXPORTER false}}
 {{$PROMETHEUS_SCRAPE_APISERVER_ONLY := DefaultParam .PROMETHEUS_SCRAPE_APISERVER_ONLY false}}
 {{$PROMETHEUS_APISERVER_SCRAPE_PORT := DefaultParam .PROMETHEUS_APISERVER_SCRAPE_PORT 443}}
-
-# Endpoints object for the kubelet running on master node.
-apiVersion: v1
-kind: Endpoints
+{{$PROMETHEUS_MASTER_DNS_ENDPOINT := DefaultParam .PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
+# EndpointSlice object for the kubelet running on master node.
+apiVersion: discovery.k8s.io/v1
+kind: EndpointSlice
 metadata:
   namespace: monitoring
   name: master
   labels:
     k8s-app: master
-subsets:
+    kubernetes.io/service-name: master
+{{if ne $PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
+addressType: FQDN
+{{else}}
+addressType: IPv4
+{{end}}
+endpoints:
+{{if ne $PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
   - addresses:
-    {{range .MasterIps}}
-      - ip: {{.}}
-    {{end}}
-    ports:
-      - name: apiserver
-        port: {{$PROMETHEUS_APISERVER_SCRAPE_PORT}}
-      {{if not $PROMETHEUS_SCRAPE_APISERVER_ONLY}}
-      - name: etcd-2379
-        port: 2379
-      - name: etcd-2382
-        port: 2382
-      - name: kubelet
-        port: 10250
-      - name: kube-scheduler
-        port: 10259
-      - name: kube-controller-manager
-        port: 10257
-      {{end}}
-      {{if $PROMETHEUS_SCRAPE_NODE_EXPORTER}}
-      - name: node-exporter
-        port: 9100
-      {{end}}
+      - {{$PROMETHEUS_MASTER_DNS_ENDPOINT}}
+{{else}}
+  {{range .MasterIps}}
+  - addresses:
+      - {{.}}
+  {{end}}
+{{end}}
+ports:
+  - name: apiserver
+    port: {{$PROMETHEUS_APISERVER_SCRAPE_PORT}}
+    protocol: TCP
+  {{if not $PROMETHEUS_SCRAPE_APISERVER_ONLY}}
+  - name: etcd-2379
+    port: 2379
+    protocol: TCP
+  - name: etcd-2382
+    port: 2382
+    protocol: TCP
+  - name: kubelet
+    port: 10250
+    protocol: TCP
+  - name: kube-scheduler
+    port: 10259
+    protocol: TCP
+  - name: kube-controller-manager
+    port: 10257
+    protocol: TCP
+  {{end}}
+  {{if $PROMETHEUS_SCRAPE_NODE_EXPORTER}}
+  - name: node-exporter
+    port: 9100
+    protocol: TCP
+  {{end}}

clusterloader2/pkg/prometheus/manifests/master-ip/master-service.yaml

@@ -1,5 +1,6 @@
 {{$PROMETHEUS_SCRAPE_NODE_EXPORTER := DefaultParam .PROMETHEUS_SCRAPE_NODE_EXPORTER false}}
 {{$PROMETHEUS_SCRAPE_APISERVER_ONLY := DefaultParam .PROMETHEUS_SCRAPE_APISERVER_ONLY false}}
+{{$PROMETHEUS_MASTER_DNS_ENDPOINT := DefaultParam .PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
 
 # Service object for the kubelet running on master node.
 apiVersion: v1
@@ -10,8 +11,13 @@ metadata:
   labels:
     k8s-app: master
 spec:
+  {{if eq $PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
   type: ClusterIP
   clusterIP: None
+  {{else}}
+  type: ExternalName
+  externalName: {{ $PROMETHEUS_MASTER_DNS_ENDPOINT }}
+  {{end}}
   ports:
     - name: apiserver
       port: 443

clusterloader2/pkg/prometheus/manifests/master-ip/master-serviceMonitor.yaml

@@ -75,4 +75,5 @@ spec:
       name: prometheus-token
       key: token
   selector:
-    k8s-app: master
+    matchLabels:
+      k8s-app: master

clusterloader2/pkg/prometheus/prometheus.go

@@ -200,6 +200,9 @@ func NewController(clusterLoaderConfig *config.ClusterLoaderConfig) (pc *Control
 	} else {
 		clusterLoaderConfig.PrometheusConfig.ScrapeNetworkPolicies = mapping["PROMETHEUS_SCRAPE_KUBE_NETWORK_POLICIES"].(bool)
 	}
+	if _, exists := mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"]; !exists {
+		mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"] = clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint
+	}
 	mapping["PROMETHEUS_SCRAPE_NODE_LOCAL_DNS"] = clusterLoaderConfig.PrometheusConfig.ScrapeNodeLocalDNS
 	mapping["PROMETHEUS_SCRAPE_KUBE_STATE_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeKubeStateMetrics
 	mapping["PROMETHEUS_SCRAPE_METRICS_SERVER_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeMetricsServerMetrics
@@ -641,6 +644,9 @@ func dumpAdditionalLogsOnPrometheusSetupFailure(k8sClient kubernetes.Interface)
 func getMasterIps(clusterConfig config.ClusterConfig, usePublicIPs bool) ([]string, error) {
 	if usePublicIPs {
 		if len(clusterConfig.MasterIPs) == 0 {
+			if clusterConfig.MasterDNSEndpoint != "" {
+				return []string{clusterConfig.MasterDNSEndpoint}, nil
+			}
 			return nil, fmt.Errorf("requested to use public IPs, however no publics IPs are provided")
 		}
 		return clusterConfig.MasterIPs, nil
@@ -649,6 +655,9 @@ func getMasterIps(clusterConfig config.ClusterConfig, usePublicIPs bool) ([]stri
 		klog.V(2).Infof("Using internal master ips (%s) to monitor master's components", clusterConfig.MasterInternalIPs)
 		return clusterConfig.MasterInternalIPs, nil
 	}
+	if clusterConfig.MasterDNSEndpoint != "" {
+		return []string{clusterConfig.MasterDNSEndpoint}, nil
+	}
 	klog.V(1).Infof("Unable to determine master ips from flags or registered nodes. Will fallback to default/kubernetes service, which can be inaccurate in HA environments.")
 	ips, err := getMasterIpsFromKubernetesService(clusterConfig)
 	if err != nil {

clusterloader2/pkg/util/cluster.go

@@ -74,17 +74,17 @@ func LogClusterNodes(c clientset.Interface) error {
 	}
 	klog.V(2).Infof("Listing cluster nodes:")
 	for i := range nodeList {
-		var internalIP, externalIP string
+		var internalIPs, externalIPs []string
 		isSchedulable := IsNodeSchedulableAndUntainted(&nodeList[i])
 		for _, address := range nodeList[i].Status.Addresses {
 			if address.Type == corev1.NodeInternalIP {
-				internalIP = address.Address
+				internalIPs = append(internalIPs, address.Address)
 			}
 			if address.Type == corev1.NodeExternalIP {
-				externalIP = address.Address
+				externalIPs = append(externalIPs, address.Address)
 			}
 		}
-		klog.V(2).Infof("Name: %v, internalIP: %v, externalIP: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIP, externalIP, isSchedulable)
+		klog.V(2).Infof("Name: %v, internalIPs: %v, externalIPs: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIPs, externalIPs, isSchedulable)
 	}
 	return nil
 }

clusterloader2/run-e2e.sh

@@ -82,11 +82,11 @@ fi
 # Create a dedicated service account for cluster-loader.
 cluster_loader_sa_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get serviceaccount cluster-loader --ignore-not-found | wc -l)
 if [[ "$cluster_loader_sa_exists" -eq 0 ]]; then
-	kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader
+   kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader
 fi
 cluster_loader_crb_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get clusterrolebinding cluster-loader --ignore-not-found | wc -l)
 if [[ "$cluster_loader_crb_exists" -eq 0 ]]; then
-	kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader
+   kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader
 fi
 
 
@@ -95,13 +95,18 @@ kubeconfig=$(mktemp)
 server=$(kubectl --kubeconfig "${KUBECONFIG}" config view -o jsonpath='{.clusters[0].cluster.server}')
 ca=$(kubectl --kubeconfig "${KUBECONFIG}" get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' | base64 -w 0)
 token=$(kubectl --kubeconfig "${KUBECONFIG}" --duration=8760h create token cluster-loader)
+ca_data=""
+
+if [[ "${MASTER_ENDPOINT:-}" == "" ]]; then
+  ca_data="    certificate-authority-data: ${ca}"
+fi
 echo "
 apiVersion: v1
 kind: Config
 clusters:
 - name: default-cluster
   cluster:
-    certificate-authority-data: ${ca}
+${ca_data}
     server: ${server}
 contexts:
 - name: default-context