k8sclient: fix per-node kubeconfig fallback

Validate the per-node kubeconfig when a current certificate is
available and fall back to the bootstrap kubeconfig only when the
per-node config is no longer trusted.

Also rebuild the derived per-node rest.Config from the reloaded
bootstrap config so TLS settings are preserved and refreshed
consistently.

Signed-off-by: Peng Liu <pliu@redhat.com>

Author: pliurh

pkg/k8sclient/kubeconfig.go

@@ -55,25 +55,29 @@ var (
 
 // getPerNodeKubeconfig creates new kubeConfig, based on bootstrap, with new certDir
 func getPerNodeKubeconfig(bootstrap *rest.Config, certDir string) *rest.Config {
-	return &rest.Config{
-		Host:    bootstrap.Host,
-		APIPath: bootstrap.APIPath,
-		ContentConfig: rest.ContentConfig{
-			AcceptContentTypes: "application/vnd.kubernetes.protobuf,application/json",
-			ContentType:        "application/vnd.kubernetes.protobuf",
-		},
-		TLSClientConfig: rest.TLSClientConfig{
-			KeyFile:  path.Join(certDir, certNamePrefix+"-current.pem"),
-			CertFile: path.Join(certDir, certNamePrefix+"-current.pem"),
-			CAData:   bootstrap.TLSClientConfig.CAData,
-		},
-		// Allow multus (especially in server mode) to make more concurrent requests
-		// to reduce client-side throttling
-		QPS:   50,
-		Burst: 50,
-		// Set the config timeout to one minute.
-		Timeout: time.Minute,
-	}
+	config := rest.CopyConfig(bootstrap)
+	config.TLSClientConfig.CertFile = path.Join(certDir, certNamePrefix+"-current.pem")
+	config.TLSClientConfig.KeyFile = path.Join(certDir, certNamePrefix+"-current.pem")
+	config.TLSClientConfig.CertData = nil
+	config.TLSClientConfig.KeyData = nil
+
+	// Switch to the per-node client certificate instead of reusing bootstrap auth.
+	config.BearerToken = ""
+	config.BearerTokenFile = ""
+	config.Username = ""
+	config.Password = ""
+	config.ExecProvider = nil
+	config.AuthProvider = nil
+
+	config.AcceptContentTypes = "application/vnd.kubernetes.protobuf,application/json"
+	config.ContentType = "application/vnd.kubernetes.protobuf"
+	// Allow multus (especially in server mode) to make more concurrent requests
+	// to reduce client-side throttling
+	config.QPS = 50
+	config.Burst = 50
+	// Set the config timeout to one minute.
+	config.Timeout = time.Minute
+	return config
 }
 
 // PerNodeK8sClient creates/reload new multus kubeconfig per-node.
@@ -84,33 +88,35 @@ func PerNodeK8sClient(nodeName, bootstrapKubeconfigFile string, certDuration tim
 	}
 	config := getPerNodeKubeconfig(bootstrapKubeconfig, certDir)
 
-	// If we have a valid certificate, user that to fetch CSRs.
+	// If we have a valid certificate, use that to fetch CSRs.
 	// Otherwise, use the bootstrap credentials from bootstrapKubeconfig
 	// https://github.com/kubernetes/kubernetes/blob/068ee321bc7bfe1c2cefb87fb4d9e5deea84fbc8/cmd/kubelet/app/server.go#L953-L963
 	newClientsetFn := func(current *tls.Certificate) (kubernetes.Interface, error) {
 		cfg := bootstrapKubeconfig
 
-		// validate the kubeconfig
-		tempClient, err := kubernetes.NewForConfig(cfg)
-		if err != nil {
-			logging.Errorf("failed to read kubeconfig from cert manager: %v", err)
-		} else {
-			_, err := tempClient.Discovery().ServerVersion()
-			// tls unknown authority error is unrecoverable error with retry
+		// When a current certificate exists, prefer it for CSR operations but fall back
+		// to bootstrap credentials if the stored per-node config is no longer trusted.
+		if current != nil {
+			cfg = config
+			tempClient, err := kubernetes.NewForConfig(cfg)
+			if err != nil {
+				return nil, logging.Errorf("failed to create client from per-node kubeconfig: %v", err)
+			}
+			_, err = tempClient.Discovery().ServerVersion()
 			if err != nil {
 				if strings.Contains(err.Error(), "x509: certificate signed by unknown authority") {
-					logging.Verbosef("cert mgr gets invalid config. rebuild from bootstrap kubeconfig")
-					// reload and use bootstrapKubeconfig again
-					newBootstrapKubeconfig, _ := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
-					cfg = newBootstrapKubeconfig
+					logging.Verbosef("cert mgr gets invalid per-node config. rebuild from bootstrap kubeconfig")
+					newBootstrapKubeconfig, reloadErr := clientcmd.BuildConfigFromFlags("", bootstrapKubeconfigFile)
+					if reloadErr != nil {
+						return nil, logging.Errorf("failed to reload bootstrap kubeconfig: %v", reloadErr)
+					}
+					bootstrapKubeconfig = newBootstrapKubeconfig
+					config = getPerNodeKubeconfig(bootstrapKubeconfig, certDir)
+					cfg = bootstrapKubeconfig
 				} else {
-					logging.Errorf("failed to validate kubeconfig with new certs: %v", err)
+					logging.Errorf("failed to validate per-node kubeconfig: %v", err)
 				}
 			}
-
-			if current != nil {
-				cfg = config
-			}
 		}
 		return kubernetes.NewForConfig(cfg)
 	}