diff --git a/clusterloader2/cmd/clusterloader.go b/clusterloader2/cmd/clusterloader.go index 95d0d1da16..291ecf2172 100644 --- a/clusterloader2/cmd/clusterloader.go +++ b/clusterloader2/cmd/clusterloader.go @@ -91,6 +91,7 @@ func initClusterFlags() { // TODO(#595): Change the name of the MASTER_IP and MASTER_INTERNAL_IP flags and vars to plural flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterIPs, "masterip", "MASTER_IP", nil /*defaultValue*/, "Hostname/IP of the master node, supports multiple values when separated by commas") flags.StringSliceEnvVar(&clusterLoaderConfig.ClusterConfig.MasterInternalIPs, "master-internal-ip", "MASTER_INTERNAL_IP", nil /*defaultValue*/, "Cluster internal/private IP of the master vm, supports multiple values when separated by commas") + flags.StringEnvVar(&clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint, "master-endpoint", "MASTER_DNS_ENDPOINT", "", "Endpoint of the master node, exclusive with --masterip and --master-internal-ips") flags.BoolEnvVar(&clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled, "apiserver-pprof-by-client-enabled", "APISERVER_PPROF_BY_CLIENT_ENABLED", true, "Whether apiserver pprof endpoint can be accessed by Kubernetes client.") flags.BoolVar(&clusterLoaderConfig.ClusterConfig.SkipClusterVerification, "skip-cluster-verification", false, "Whether to skip the cluster verification, which expects at least one schedulable node in the cluster") @@ -167,6 +168,23 @@ func completeConfig(m *framework.MultiClientSet) error { clusterLoaderConfig.ClusterConfig.Nodes = nodes klog.V(0).Infof("ClusterConfig.Nodes set to %v", nodes) } + if clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint == "" { + err := completeIpMasterConfig(m) + if err != nil { + return err + } + } + + if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint { + clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false + } + if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 { + clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes) + } + return nil +} + +func completeIpMasterConfig(m *framework.MultiClientSet) error { if clusterLoaderConfig.ClusterConfig.MasterName == "" { masterName, err := util.GetMasterName(m.GetClient()) if err == nil { @@ -194,13 +212,6 @@ func completeConfig(m *framework.MultiClientSet) error { klog.Errorf("Getting master internal ip error: %v", err) } } - - if !clusterLoaderConfig.ClusterConfig.Provider.Features().SupportAccessAPIServerPprofEndpoint { - clusterLoaderConfig.ClusterConfig.APIServerPprofByClientEnabled = false - } - if clusterLoaderConfig.ClusterConfig.K8SClientsNumber == 0 { - clusterLoaderConfig.ClusterConfig.K8SClientsNumber = getClientsNumber(clusterLoaderConfig.ClusterConfig.Nodes) - } return nil } @@ -288,6 +299,15 @@ func main() { klog.Exitf("Parsing flags error: %v", errList.String()) } + klog.V(2).Infof("KubeConfigPath: %v", clusterLoaderConfig.ClusterConfig.KubeConfigPath) + if clusterLoaderConfig.ClusterConfig.KubeConfigPath != "" { + content, err := ioutil.ReadFile(clusterLoaderConfig.ClusterConfig.KubeConfigPath) + if err != nil { + klog.Errorf("Error reading kubeconfig: %v", err) + } else { + klog.V(2).Infof("KubeConfig content:\n%s", string(content)) + } + } mclient, err := framework.NewMultiClientSet(clusterLoaderConfig.ClusterConfig.KubeConfigPath, 1) if err != nil { klog.Exitf("Client creation error: %v", err) diff --git a/clusterloader2/pkg/config/cluster.go b/clusterloader2/pkg/config/cluster.go index 0776587fb3..ea775bf316 100644 --- a/clusterloader2/pkg/config/cluster.go +++ b/clusterloader2/pkg/config/cluster.go @@ -46,6 +46,7 @@ type ClusterConfig struct { MasterIPs []string MasterInternalIPs []string MasterName string + MasterDNSEndpoint string // Deprecated: use NamespaceConfig.DeleteStaleNamespaces instead. DeleteStaleNamespaces bool // TODO(#1696): Clean up after removing automanagedNamespaces diff --git a/clusterloader2/pkg/measurement/common/profile.go b/clusterloader2/pkg/measurement/common/profile.go index 7d638669e5..133b7fa35c 100644 --- a/clusterloader2/pkg/measurement/common/profile.go +++ b/clusterloader2/pkg/measurement/common/profile.go @@ -67,6 +67,9 @@ func (p *profileMeasurement) populateProfileConfig(config *measurement.Config) e } p.config.provider = config.ClusterFramework.GetClusterConfig().Provider p.config.hosts = config.ClusterFramework.GetClusterConfig().MasterIPs + if masterDnsEnpoint := config.ClusterFramework.GetClusterConfig().MasterDNSEndpoint; masterDnsEnpoint != "" { + p.config.hosts = append(p.config.hosts, masterDnsEnpoint) + } return nil } diff --git a/clusterloader2/pkg/prometheus/manifests/master-ip/master-endpoints.yaml b/clusterloader2/pkg/prometheus/manifests/master-ip/master-endpoints.yaml index d23e3bf561..e0e78abeee 100644 --- a/clusterloader2/pkg/prometheus/manifests/master-ip/master-endpoints.yaml +++ b/clusterloader2/pkg/prometheus/manifests/master-ip/master-endpoints.yaml @@ -1,36 +1,54 @@ {{$PROMETHEUS_SCRAPE_NODE_EXPORTER := DefaultParam .PROMETHEUS_SCRAPE_NODE_EXPORTER false}} {{$PROMETHEUS_SCRAPE_APISERVER_ONLY := DefaultParam .PROMETHEUS_SCRAPE_APISERVER_ONLY false}} {{$PROMETHEUS_APISERVER_SCRAPE_PORT := DefaultParam .PROMETHEUS_APISERVER_SCRAPE_PORT 443}} - -# Endpoints object for the kubelet running on master node. -apiVersion: v1 -kind: Endpoints +{{$PROMETHEUS_MASTER_DNS_ENDPOINT := DefaultParam .PROMETHEUS_MASTER_DNS_ENDPOINT ""}} +# EndpointSlice object for the kubelet running on master node. +apiVersion: discovery.k8s.io/v1 +kind: EndpointSlice metadata: namespace: monitoring name: master labels: k8s-app: master -subsets: + kubernetes.io/service-name: master +{{if ne $PROMETHEUS_MASTER_DNS_ENDPOINT ""}} +addressType: FQDN +{{else}} +addressType: IPv4 +{{end}} +endpoints: +{{if ne $PROMETHEUS_MASTER_DNS_ENDPOINT ""}} - addresses: - {{range .MasterIps}} - - ip: {{.}} - {{end}} - ports: - - name: apiserver - port: {{$PROMETHEUS_APISERVER_SCRAPE_PORT}} - {{if not $PROMETHEUS_SCRAPE_APISERVER_ONLY}} - - name: etcd-2379 - port: 2379 - - name: etcd-2382 - port: 2382 - - name: kubelet - port: 10250 - - name: kube-scheduler - port: 10259 - - name: kube-controller-manager - port: 10257 - {{end}} - {{if $PROMETHEUS_SCRAPE_NODE_EXPORTER}} - - name: node-exporter - port: 9100 - {{end}} + - {{$PROMETHEUS_MASTER_DNS_ENDPOINT}} +{{else}} + {{range .MasterIps}} + - addresses: + - {{.}} + {{end}} +{{end}} +ports: + - name: apiserver + port: {{$PROMETHEUS_APISERVER_SCRAPE_PORT}} + protocol: TCP + {{if not $PROMETHEUS_SCRAPE_APISERVER_ONLY}} + - name: etcd-2379 + port: 2379 + protocol: TCP + - name: etcd-2382 + port: 2382 + protocol: TCP + - name: kubelet + port: 10250 + protocol: TCP + - name: kube-scheduler + port: 10259 + protocol: TCP + - name: kube-controller-manager + port: 10257 + protocol: TCP + {{end}} + {{if $PROMETHEUS_SCRAPE_NODE_EXPORTER}} + - name: node-exporter + port: 9100 + protocol: TCP + {{end}} \ No newline at end of file diff --git a/clusterloader2/pkg/prometheus/manifests/master-ip/master-service.yaml b/clusterloader2/pkg/prometheus/manifests/master-ip/master-service.yaml index 4a5ca7846f..3e4b080023 100644 --- a/clusterloader2/pkg/prometheus/manifests/master-ip/master-service.yaml +++ b/clusterloader2/pkg/prometheus/manifests/master-ip/master-service.yaml @@ -1,5 +1,6 @@ {{$PROMETHEUS_SCRAPE_NODE_EXPORTER := DefaultParam .PROMETHEUS_SCRAPE_NODE_EXPORTER false}} {{$PROMETHEUS_SCRAPE_APISERVER_ONLY := DefaultParam .PROMETHEUS_SCRAPE_APISERVER_ONLY false}} +{{$PROMETHEUS_MASTER_DNS_ENDPOINT := DefaultParam .PROMETHEUS_MASTER_DNS_ENDPOINT ""}} # Service object for the kubelet running on master node. apiVersion: v1 @@ -10,8 +11,13 @@ metadata: labels: k8s-app: master spec: + {{if eq $PROMETHEUS_MASTER_DNS_ENDPOINT ""}} type: ClusterIP clusterIP: None + {{else}} + type: ExternalName + externalName: {{ $PROMETHEUS_MASTER_DNS_ENDPOINT }} + {{end}} ports: - name: apiserver port: 443 diff --git a/clusterloader2/pkg/prometheus/manifests/master-ip/master-serviceMonitor.yaml b/clusterloader2/pkg/prometheus/manifests/master-ip/master-serviceMonitor.yaml index 3c37997423..879a3cf3b4 100644 --- a/clusterloader2/pkg/prometheus/manifests/master-ip/master-serviceMonitor.yaml +++ b/clusterloader2/pkg/prometheus/manifests/master-ip/master-serviceMonitor.yaml @@ -75,4 +75,5 @@ spec: name: prometheus-token key: token selector: - k8s-app: master + matchLabels: + k8s-app: master diff --git a/clusterloader2/pkg/prometheus/prometheus.go b/clusterloader2/pkg/prometheus/prometheus.go index 7eb0918b62..2452f969c3 100644 --- a/clusterloader2/pkg/prometheus/prometheus.go +++ b/clusterloader2/pkg/prometheus/prometheus.go @@ -200,6 +200,9 @@ func NewController(clusterLoaderConfig *config.ClusterLoaderConfig) (pc *Control } else { clusterLoaderConfig.PrometheusConfig.ScrapeNetworkPolicies = mapping["PROMETHEUS_SCRAPE_KUBE_NETWORK_POLICIES"].(bool) } + if _, exists := mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"]; !exists { + mapping["PROMETHEUS_MASTER_DNS_ENDPOINT"] = clusterLoaderConfig.ClusterConfig.MasterDNSEndpoint + } mapping["PROMETHEUS_SCRAPE_NODE_LOCAL_DNS"] = clusterLoaderConfig.PrometheusConfig.ScrapeNodeLocalDNS mapping["PROMETHEUS_SCRAPE_KUBE_STATE_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeKubeStateMetrics mapping["PROMETHEUS_SCRAPE_METRICS_SERVER_METRICS"] = clusterLoaderConfig.PrometheusConfig.ScrapeMetricsServerMetrics @@ -641,6 +644,9 @@ func dumpAdditionalLogsOnPrometheusSetupFailure(k8sClient kubernetes.Interface) func getMasterIps(clusterConfig config.ClusterConfig, usePublicIPs bool) ([]string, error) { if usePublicIPs { if len(clusterConfig.MasterIPs) == 0 { + if clusterConfig.MasterDNSEndpoint != "" { + return []string{clusterConfig.MasterDNSEndpoint}, nil + } return nil, fmt.Errorf("requested to use public IPs, however no publics IPs are provided") } return clusterConfig.MasterIPs, nil @@ -649,6 +655,9 @@ func getMasterIps(clusterConfig config.ClusterConfig, usePublicIPs bool) ([]stri klog.V(2).Infof("Using internal master ips (%s) to monitor master's components", clusterConfig.MasterInternalIPs) return clusterConfig.MasterInternalIPs, nil } + if clusterConfig.MasterDNSEndpoint != "" { + return []string{clusterConfig.MasterDNSEndpoint}, nil + } klog.V(1).Infof("Unable to determine master ips from flags or registered nodes. Will fallback to default/kubernetes service, which can be inaccurate in HA environments.") ips, err := getMasterIpsFromKubernetesService(clusterConfig) if err != nil { diff --git a/clusterloader2/pkg/util/cluster.go b/clusterloader2/pkg/util/cluster.go index f3ebdbbf74..6529d8bf3d 100644 --- a/clusterloader2/pkg/util/cluster.go +++ b/clusterloader2/pkg/util/cluster.go @@ -74,17 +74,17 @@ func LogClusterNodes(c clientset.Interface) error { } klog.V(2).Infof("Listing cluster nodes:") for i := range nodeList { - var internalIP, externalIP string + var internalIPs, externalIPs []string isSchedulable := IsNodeSchedulableAndUntainted(&nodeList[i]) for _, address := range nodeList[i].Status.Addresses { if address.Type == corev1.NodeInternalIP { - internalIP = address.Address + internalIPs = append(internalIPs, address.Address) } if address.Type == corev1.NodeExternalIP { - externalIP = address.Address + externalIPs = append(externalIPs, address.Address) } } - klog.V(2).Infof("Name: %v, internalIP: %v, externalIP: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIP, externalIP, isSchedulable) + klog.V(2).Infof("Name: %v, internalIPs: %v, externalIPs: %v, isSchedulable: %v", nodeList[i].ObjectMeta.Name, internalIPs, externalIPs, isSchedulable) } return nil } diff --git a/clusterloader2/run-e2e.sh b/clusterloader2/run-e2e.sh index 748ab4622a..fc304548fc 100755 --- a/clusterloader2/run-e2e.sh +++ b/clusterloader2/run-e2e.sh @@ -82,11 +82,11 @@ fi # Create a dedicated service account for cluster-loader. cluster_loader_sa_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get serviceaccount cluster-loader --ignore-not-found | wc -l) if [[ "$cluster_loader_sa_exists" -eq 0 ]]; then - kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader + kubectl --kubeconfig "${KUBECONFIG}" create serviceaccount cluster-loader fi cluster_loader_crb_exists=$(kubectl --kubeconfig "${KUBECONFIG}" get clusterrolebinding cluster-loader --ignore-not-found | wc -l) if [[ "$cluster_loader_crb_exists" -eq 0 ]]; then - kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader + kubectl --kubeconfig "${KUBECONFIG}" create clusterrolebinding cluster-loader --clusterrole=cluster-admin --serviceaccount=default:cluster-loader fi @@ -95,13 +95,18 @@ kubeconfig=$(mktemp) server=$(kubectl --kubeconfig "${KUBECONFIG}" config view -o jsonpath='{.clusters[0].cluster.server}') ca=$(kubectl --kubeconfig "${KUBECONFIG}" get configmap kube-root-ca.crt -o jsonpath='{.data.ca\.crt}' | base64 -w 0) token=$(kubectl --kubeconfig "${KUBECONFIG}" --duration=8760h create token cluster-loader) +ca_data="" + +if [[ "${MASTER_ENDPOINT:-}" == "" ]]; then + ca_data=" certificate-authority-data: ${ca}" +fi echo " apiVersion: v1 kind: Config clusters: - name: default-cluster cluster: - certificate-authority-data: ${ca} +${ca_data} server: ${server} contexts: - name: default-context