Skip to content

Commit 9e2edca

Browse files
matix522matix522
committed
feat: Add an ability to measure clusters with dns only control plane access
This update allows ClusterLoader2 to target clusters using a DNS endpoint for the control plane. It includes support for the MASTER_DNS_ENDPOINT environment variable and relevant configuration changes. Changed logging to show all of the Node adresses to allow for dualstack clusters.
1 parent a32966a commit 9e2edca

File tree

9 files changed

+105
-42
lines changed

9 files changed

+105
-42
lines changed

‎clusterloader2/cmd/clusterloader.go‎

Lines changed: 27 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@ func initClusterFlags() {
9191
// TODO(#595): Change the name of the MASTER_IP and MASTER_INTERNAL_IP flags and vars to plural
9292
flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterIPs, "masterip", "MASTER_IP", nil /*defaultValue*/, "Hostname/IP of the master node, supports multiple values when separated by commas")
9393
flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterInternalIPs, "master-internal-ip", "MASTER_INTERNAL_IP", nil /*defaultValue*/, "Cluster internal/private IP of the master vm, supports multiple values when separated by commas")
94+
flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint, "master-endpoint", "MASTER_DNS_ENDPOINT", "", "Endpoint of the master node, exclusive with --masterip and --master-internal-ips")
9495
flags.BoolEnvVar(&clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled, "apiserver-pprof-by-client-enabled", "APISERVER_PPROF_BY_CLIENT_ENABLED", true, "Whether apiserver pprof endpoint can be accessed by Kubernetes client.")
9596
flags.BoolVar(&clusterLoaderConfig.ClusterConfig.SkipClusterVerification, "skip-cluster-verification", false, "Whether to skip the cluster verification, which expects at least one schedulable node in the cluster")
9697

@@ -167,6 +168,23 @@ func completeConfig(m *framework.MultiClientSet) error {
167168
clusterLoaderConfig.ClusterConfig.Nodes = nodes
168169
klog.V(0).Infof("ClusterConfig.Nodes set to %v", nodes)
169170
}
171+
if clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint == "" {
172+
err := completeIpMasterConfig(m)
173+
if err != nil {
174+
return err
175+
}
176+
}
177+
178+
if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint {
179+
clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false
180+
}
181+
if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 {
182+
clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes)
183+
}
184+
return nil
185+
}
186+
187+
func completeIpMasterConfig(m *framework.MultiClientSet) error {
170188
if clusterLoaderConfig.ClusterConfig.MasterName == "" {
171189
masterName, err := util.GetMasterName(m.GetClient())
172190
if err == nil {
@@ -194,13 +212,6 @@ func completeConfig(m *framework.MultiClientSet) error {
194212
klog.Errorf("Getting master internal ip error: %v", err)
195213
}
196214
}
197-
198-
if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint {
199-
clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false
200-
}
201-
if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 {
202-
clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes)
203-
}
204215
return nil
205216
}
206217

@@ -288,6 +299,15 @@ func main() {
288299
klog.Exitf("Parsing flags error: %v", errList.String())
289300
}
290301

302+
klog.V(2).Infof("KubeConfigPath: %v", clusterLoaderConfig.ClusterConfig.KubeConfigPath)
303+
if clusterLoaderConfig.ClusterConfig.KubeConfigPath != "" {
304+
content, err := ioutil.ReadFile(clusterLoaderConfig.ClusterConfig.KubeConfigPath)
305+
if err != nil {
306+
klog.Errorf("Error reading kubeconfig: %v", err)
307+
} else {
308+
klog.V(2).Infof("KubeConfig content:\n%s", string(content))
309+
}
310+
}
291311
mclient, err := framework.NewMultiClientSet(clusterLoaderConfig.ClusterConfig.KubeConfigPath, 1)
292312
if err != nil {
293313
klog.Exitf("Client creation error: %v", err)

‎clusterloader2/pkg/config/cluster.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ type ClusterConfig struct {
4646
MasterIPs []string
4747
MasterInternalIPs []string
4848
MasterName string
49+
MasterDNSEndpoint string
4950
// Deprecated: use NamespaceConfig.DeleteStaleNamespaces instead.
5051
DeleteStaleNamespaces bool
5152
// TODO(#1696): Clean up after removing automanagedNamespaces

‎clusterloader2/pkg/measurement/common/profile.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,9 @@ func (p *profileMeasurement) populateProfileConfig(config *measurement.Config) e
6767
}
6868
p.config.provider = config.ClusterFramework.GetClusterConfig().Provider
6969
p.config.hosts = config.ClusterFramework.GetClusterConfig().MasterIPs
70+
if masterDnsEnpoint := config.ClusterFramework.GetClusterConfig().MasterDNSEndpoint; masterDnsEnpoint != "" {
71+
p.config.hosts = append(p.config.hosts, masterDnsEnpoint)
72+
}
7073
return nil
7174
}
7275

‎clusterloader2/pkg/prometheus/manifests/master-ip/master-endpoints.yaml‎

Lines changed: 45 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,54 @@
11
{{$PROMETHEUS_SCRAPE_NODE_EXPORTER := DefaultParam .PROMETHEUS_SCRAPE_NODE_EXPORTER false}}
22
{{$PROMETHEUS_SCRAPE_APISERVER_ONLY := DefaultParam .PROMETHEUS_SCRAPE_APISERVER_ONLY false}}
33
{{$PROMETHEUS_APISERVER_SCRAPE_PORT := DefaultParam .PROMETHEUS_APISERVER_SCRAPE_PORT 443}}
4-
5-
# Endpoints object for the kubelet running on master node.
6-
apiVersion: v1
7-
kind: Endpoints
4+
{{$PROMETHEUS_MASTER_DNS_ENDPOINT := DefaultParam .PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
5+
# EndpointSlice object for the kubelet running on master node.
6+
apiVersion: discovery.k8s.io/v1
7+
kind: EndpointSlice
88
metadata:
99
namespace: monitoring
1010
name: master
1111
labels:
1212
k8s-app: master
13-
subsets:
13+
kubernetes.io/service-name: master
14+
{{if ne $PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
15+
addressType: FQDN
16+
{{else}}
17+
addressType: IPv4
18+
{{end}}
19+
endpoints:
20+
{{if ne $PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
1421
- addresses:
15-
{{range .MasterIps}}
16-
- ip: {{.}}
17-
{{end}}
18-
ports:
19-
- name: apiserver
20-
port: {{$PROMETHEUS_APISERVER_SCRAPE_PORT}}
21-
{{if not $PROMETHEUS_SCRAPE_APISERVER_ONLY}}
22-
- name: etcd-2379
23-
port: 2379
24-
- name: etcd-2382
25-
port: 2382
26-
- name: kubelet
27-
port: 10250
28-
- name: kube-scheduler
29-
port: 10259
30-
- name: kube-controller-manager
31-
port: 10257
32-
{{end}}
33-
{{if $PROMETHEUS_SCRAPE_NODE_EXPORTER}}
34-
- name: node-exporter
35-
port: 9100
36-
{{end}}
22+
- {{$PROMETHEUS_MASTER_DNS_ENDPOINT}}
23+
{{else}}
24+
{{range .MasterIps}}
25+
- addresses:
26+
- {{.}}
27+
{{end}}
28+
{{end}}
29+
ports:
30+
- name: apiserver
31+
port: {{$PROMETHEUS_APISERVER_SCRAPE_PORT}}
32+
protocol: TCP
33+
{{if not $PROMETHEUS_SCRAPE_APISERVER_ONLY}}
34+
- name: etcd-2379
35+
port: 2379
36+
protocol: TCP
37+
- name: etcd-2382
38+
port: 2382
39+
protocol: TCP
40+
- name: kubelet
41+
port: 10250
42+
protocol: TCP
43+
- name: kube-scheduler
44+
port: 10259
45+
protocol: TCP
46+
- name: kube-controller-manager
47+
port: 10257
48+
protocol: TCP
49+
{{end}}
50+
{{if $PROMETHEUS_SCRAPE_NODE_EXPORTER}}
51+
- name: node-exporter
52+
port: 9100
53+
protocol: TCP
54+
{{end}}

‎clusterloader2/pkg/prometheus/manifests/master-ip/master-service.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
{{$PROMETHEUS_SCRAPE_NODE_EXPORTER := DefaultParam .PROMETHEUS_SCRAPE_NODE_EXPORTER false}}
22
{{$PROMETHEUS_SCRAPE_APISERVER_ONLY := DefaultParam .PROMETHEUS_SCRAPE_APISERVER_ONLY false}}
3+
{{$PROMETHEUS_MASTER_DNS_ENDPOINT := DefaultParam .PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
34

45
# Service object for the kubelet running on master node.
56
apiVersion: v1
@@ -10,8 +11,13 @@ metadata:
1011
labels:
1112
k8s-app: master
1213
spec:
14+
{{if eq $PROMETHEUS_MASTER_DNS_ENDPOINT ""}}
1315
type: ClusterIP
1416
clusterIP: None
17+
{{else}}
18+
type: ExternalName
19+
externalName: {{ $PROMETHEUS_MASTER_DNS_ENDPOINT }}
20+
{{end}}
1521
ports:
1622
- name: apiserver
1723
port: 443

‎clusterloader2/pkg/prometheus/manifests/master-ip/master-serviceMonitor.yaml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,4 +75,5 @@ spec:
7575
name: prometheus-token
7676
key: token
7777
selector:
78-
k8s-app: master
78+
matchLabels:
79+
k8s-app: master

‎clusterloader2/pkg/prometheus/prometheus.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,9 @@ func NewController(clusterLoaderConfig *config.ClusterLoaderConfig) (pc *Control
200200
} else {
201201
clusterLoaderConfig.PrometheusConfig.ScrapeNetworkPolicies = mapping["PROMETHEUS_SCRAPE_KUBE_NETWORK_POLICIES"].(bool)
202202
}
203+
if _, exists := mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"]; !exists {
204+
mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"] = clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint
205+
}
203206
mapping["PROMETHEUS_SCRAPE_NODE_LOCAL_DNS"] = clusterLoaderConfig.PrometheusConfig.ScrapeNodeLocalDNS
204207
mapping["PROMETHEUS_SCRAPE_KUBE_STATE_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeKubeStateMetrics
205208
mapping["PROMETHEUS_SCRAPE_METRICS_SERVER_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeMetricsServerMetrics
@@ -641,6 +644,9 @@ func dumpAdditionalLogsOnPrometheusSetupFailure(k8sClient kubernetes.Interface)
641644
func getMasterIps(clusterConfig config.ClusterConfig, usePublicIPs bool) ([]string, error) {
642645
if usePublicIPs {
643646
if len(clusterConfig.MasterIPs) == 0 {
647+
if clusterConfig.MasterDNSEndpoint != "" {
648+
return []string{clusterConfig.MasterDNSEndpoint}, nil
649+
}
644650
return nil, fmt.Errorf("requested to use public IPs, however no publics IPs are provided")
645651
}
646652
return clusterConfig.MasterIPs, nil
@@ -649,6 +655,9 @@ func getMasterIps(clusterConfig config.ClusterConfig, usePublicIPs bool) ([]stri
649655
klog.V(2).Infof("Using internal master ips (%s) to monitor master's components", clusterConfig.MasterInternalIPs)
650656
return clusterConfig.MasterInternalIPs, nil
651657
}
658+
if clusterConfig.MasterDNSEndpoint != "" {
659+
return []string{clusterConfig.MasterDNSEndpoint}, nil
660+
}
652661
klog.V(1).Infof("Unable to determine master ips from flags or registered nodes. Will fallback to default/kubernetes service, which can be inaccurate in HA environments.")
653662
ips, err := getMasterIpsFromKubernetesService(clusterConfig)
654663
if err != nil {

‎clusterloader2/pkg/util/cluster.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -74,17 +74,17 @@ func LogClusterNodes(c clientset.Interface) error {
7474
}
7575
klog.V(2).Infof("Listing cluster nodes:")
7676
for i := range nodeList {
77-
var internalIP, externalIP string
77+
var internalIPs, externalIPs []string
7878
isSchedulable := IsNodeSchedulableAndUntainted(&nodeList[i])
7979
for _, address := range nodeList[i].Status.Addresses {
8080
if address.Type == corev1.NodeInternalIP {
81-
internalIP = address.Address
81+
internalIPs = append(internalIPs, address.Address)
8282
}
8383
if address.Type == corev1.NodeExternalIP {
84-
externalIP = address.Address
84+
externalIPs = append(externalIPs, address.Address)
8585
}
8686
}
87-
klog.V(2).Infof("Name: %v, internalIP: %v, externalIP: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIP, externalIP, isSchedulable)
87+
klog.V(2).Infof("Name: %v, internalIPs: %v, externalIPs: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIPs, externalIPs, isSchedulable)
8888
}
8989
return nil
9090
}

‎clusterloader2/run-e2e.sh‎

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,11 +82,11 @@ fi
8282
# Create a dedicated service account for cluster-loader.
8383
cluster_loader_sa_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get serviceaccount cluster-loader --ignore-not-found | wc -l)
8484
if [[ "$cluster_loader_sa_exists" -eq 0 ]]; then
85-
kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader
85+
kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader
8686
fi
8787
cluster_loader_crb_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get clusterrolebinding cluster-loader --ignore-not-found | wc -l)
8888
if [[ "$cluster_loader_crb_exists" -eq 0 ]]; then
89-
kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader
89+
kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader
9090
fi
9191

9292

@@ -95,13 +95,18 @@ kubeconfig=$(mktemp)
9595
server=$(kubectl --kubeconfig "${KUBECONFIG}" config view -o jsonpath='{.clusters[0].cluster.server}')
9696
ca=$(kubectl --kubeconfig "${KUBECONFIG}" get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' | base64 -w 0)
9797
token=$(kubectl --kubeconfig "${KUBECONFIG}" --duration=8760h create token cluster-loader)
98+
ca_data=""
99+
100+
if [[ "${MASTER_ENDPOINT:-}" == "" ]]; then
101+
ca_data=" certificate-authority-data: ${ca}"
102+
fi
98103
echo "
99104
apiVersion: v1
100105
kind: Config
101106
clusters:
102107
- name: default-cluster
103108
cluster:
104-
certificate-authority-data: ${ca}
109+
${ca_data}
105110
server: ${server}
106111
contexts:
107112
- name: default-context

0 commit comments

Comments
 (0)