Merge pull request #837 from rawmind0/kubeconfig

Fix kube_config generation function for Rancher 2.6.0 and above

Author: rawmind0

CHANGELOG.md

@@ -1,4 +1,4 @@
-## 1.22.2 (Unreleased)
+## 1.22.2 (January 6, 2022)
 
 FEATURES:
 
@@ -12,6 +12,7 @@ BUG FIXES:
 
 * Fix `rancher2_cluster_sync.state_confirm` behaviour https://github.com/rancher/terraform-provider-rancher2/issues/797
 * Fix `rancher2_cluster` monitoring flips on resource update https://github.com/rancher/terraform-provider-rancher2/issues/825
+* Fix kube_config generation function at `rancher2_cluster`, `rancher2_cluster_v2` and `rancher2_cluster_sync` for Rancher 2.6.0 and above https://github.com/rancher/terraform-provider-rancher2/issues/789
 
 ## 1.22.1 (December 23, 2021)
 

docs/resources/cluster.md

@@ -620,6 +620,8 @@ The following attributes are exported:
 * `ca_cert` - (Computed/Sensitive) K8s cluster ca cert (string)
 * `system_project_id` - (Computed) System project ID for the cluster (string)
 
+**Note** For Rancher 2.6.0 and above: if setting `kubeconfig-generate-token=false` then the generated `kube_config` will not contain any user token. `kubectl` will generate the user token executing the [rancher cli](https://github.com/rancher/cli/releases/tag/v2.6.0), so it should be installed previously.
+
 ## Nested blocks
 
 ### `agent_env_vars`

docs/resources/cluster_sync.md

@@ -103,6 +103,8 @@ The following attributes are exported:
 * `nodes` - (Computed) The cluster nodes (list).
 * `system_project_id` - (Computed) System project ID for the cluster sync (string)
 
+**Note** For Rancher 2.6.0 and above: if setting `kubeconfig-generate-token=false` then the generated `kube_config` will not contain any user token. `kubectl` will generate the user token executing the [rancher cli](https://github.com/rancher/cli/releases/tag/v2.6.0), so it should be installed previously.
+
 ## Nested blocks
 
 ### `nodes`

docs/resources/cluster_v2.md

@@ -234,6 +234,8 @@ The following attributes are exported:
 * `cluster_v1_id` - (Computed) Cluster v1 id for cluster v2. (e.g to be used with `rancher2_sync`) (string)
 * `resource_version` - (Computed) Cluster v2 k8s resource version (string)
 
+**Note** For Rancher 2.6.0 and above: if setting `kubeconfig-generate-token=false` then the generated `kube_config` will not contain any user token. `kubectl` will generate the user token executing the [rancher cli](https://github.com/rancher/cli/releases/tag/v2.6.0), so it should be installed previously.
+
 ## Nested blocks
 
 ### `agent_env_vars`

go.mod

@@ -16,6 +16,7 @@ require (
 	k8s.io/api v0.22.3
 	k8s.io/apimachinery v0.22.3
 	k8s.io/apiserver v0.22.3
+	k8s.io/client-go v12.0.0+incompatible
 )
 
 replace (

rancher2/resource_rancher2_cluster.go

@@ -12,6 +12,8 @@ import (
 	norman "github.com/rancher/norman/types"
 	managementClient "github.com/rancher/rancher/pkg/client/generated/management/v3"
 	projectClient "github.com/rancher/rancher/pkg/client/generated/project/v3"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 )
 
 func resourceRancher2Cluster() *schema.Resource {
@@ -238,7 +240,7 @@ func resourceRancher2ClusterRead(d *schema.ResourceData, meta interface{}) error
 		return err
 	}
 
-	kubeConfig, err := getClusterKubeconfig(meta.(*Config), cluster.ID)
+	kubeConfig, err := getClusterKubeconfig(meta.(*Config), cluster.ID, d.Get("kube_config").(string))
 	if err != nil && !IsForbidden(err) {
 		return err
 	}
@@ -584,10 +586,81 @@ func createClusterRegistrationToken(client *managementClient.Client, clusterID s
 	return newRegToken, nil
 }
 
-func getClusterKubeconfig(c *Config, id string) (*managementClient.GenerateKubeConfigOutput, error) {
+func isKubeConfigValid(c *Config, config string) (string, bool, error) {
+	token, tokenValid, err := isKubeConfigTokenValid(c, config)
+	if err != nil {
+		return "", false, err
+	}
+	if !tokenValid {
+		return "", false, nil
+	}
+	kubeconfig, err := clientcmd.RESTConfigFromKubeConfig([]byte(config))
+	if err != nil {
+		return "", false, fmt.Errorf("Checking Kubeconfig: %v", err)
+	}
+	_, err = kubernetes.NewForConfig(kubeconfig)
+	if err != nil {
+		return token, false, nil
+	}
+
+	return token, true, nil
+}
+
+func isKubeConfigTokenValid(c *Config, config string) (string, bool, error) {
+	token, err := getTokenFromKubeConfig(config)
+	if err != nil {
+		return "", false, fmt.Errorf("Getting Kubeconfig token: %v", err)
+	}
+	isValid, err := isTokenValid(c, splitTokenID(token))
+	if err != nil {
+		return "", false, fmt.Errorf("Checking Kubeconfig token: %v", err)
+	}
+	return token, isValid, nil
+}
+
+func replaceKubeConfigToken(c *Config, config, token string) (string, error) {
+	if len(token) == 0 {
+		return config, nil
+	}
+	kubeconfig, err := getObjFromKubeConfig(config)
+	if err != nil {
+		return "", fmt.Errorf("Getting K8s config object: %v", err)
+	}
+	if kubeconfig == nil || kubeconfig.AuthInfos == nil || len(kubeconfig.AuthInfos) == 0 {
+		return config, nil
+	}
+
+	client, err := c.ManagementClient()
+	if err != nil {
+		return "", fmt.Errorf("Replacing cluster Kubeconfig token: %v", err)
+	}
+	removeToken, err := client.Token.ByID(splitTokenID(kubeconfig.AuthInfos[0].AuthInfo.Token))
+	if err != nil {
+		if !IsNotFound(err) && !IsForbidden(err) {
+			return "", err
+		}
+	}
+
+	err = client.Token.Delete(removeToken)
+	if err != nil {
+		return "", fmt.Errorf("Error removing Token: %s", err)
+	}
+	kubeconfig.AuthInfos[0].AuthInfo.Token = token
+	return getKubeConfigFromObj(kubeconfig)
+}
+
+func getClusterKubeconfig(c *Config, id, origconfig string) (*managementClient.GenerateKubeConfigOutput, error) {
 	action := "generateKubeconfig"
 	cluster := &Cluster{}
 
+	token, kubeValid, err := isKubeConfigValid(c, origconfig)
+	if err != nil {
+		return nil, fmt.Errorf("Getting cluster Kubeconfig: %v", err)
+	}
+	if kubeValid {
+		return &managementClient.GenerateKubeConfigOutput{Config: origconfig}, nil
+	}
+
 	client, err := c.ManagementClient()
 	if err != nil {
 		return nil, fmt.Errorf("Getting cluster Kubeconfig: %v", err)
@@ -621,6 +694,13 @@ func getClusterKubeconfig(c *Config, id string) (*managementClient.GenerateKubeC
 			}
 			err = client.APIBaseClient.Action(managementClient.ClusterType, action, clusterResource, nil, kubeConfig)
 			if err == nil {
+				if isRancher26 && len(token) > 0 {
+					newConfig, err := replaceKubeConfigToken(c, kubeConfig.Config, token)
+					if err != nil {
+						return nil, err
+					}
+					kubeConfig.Config = newConfig
+				}
 				return kubeConfig, nil
 			}
 			if !IsNotFound(err) && !IsForbidden(err) && !IsServiceUnavailableError(err) {

rancher2/resource_rancher2_cluster_sync.go

@@ -88,11 +88,7 @@ func resourceRancher2ClusterSyncRead(d *schema.ResourceData, meta interface{}) e
 		d.Set("default_project_id", defaultProjectID)
 		d.Set("system_project_id", systemProjectID)
 
-		client, err := meta.(*Config).ManagementClient()
-		if err != nil {
-			return err
-		}
-		kubeConfig, err := client.Cluster.ActionGenerateKubeconfig(clus)
+		kubeConfig, err := getClusterKubeconfig(meta.(*Config), clusterID, d.Get("kube_config").(string))
 		if err != nil {
 			return err
 		}

rancher2/resource_rancher2_cluster_v2.go

@@ -342,7 +342,7 @@ func setClusterV2LegacyData(d *schema.ResourceData, c *Config) error {
 		return fmt.Errorf("Setting cluster V2 legacy data: %v", err)
 	}
 
-	kubeConfig, err := getClusterKubeconfig(c, cluster.ID)
+	kubeConfig, err := getClusterKubeconfig(c, cluster.ID, d.Get("kube_config").(string))
 	if err != nil {
 		return fmt.Errorf("Setting cluster V2 legacy data: %v", err)
 	}

rancher2/resource_rancher2_token.go

@@ -117,3 +117,24 @@ func resourceRancher2TokenDelete(d *schema.ResourceData, meta interface{}) error
 	d.SetId("")
 	return nil
 }
+
+func isTokenValid(c *Config, id string) (bool, error) {
+	if len(id) == 0 {
+		return false, nil
+	}
+
+	client, err := c.ManagementClient()
+	if err != nil {
+		return false, err
+	}
+
+	token, err := client.Token.ByID(id)
+	if err != nil {
+		if !IsNotFound(err) && !IsForbidden(err) {
+			return false, err
+		}
+		return false, nil
+	}
+	// Token is valid if it's enabled and not expired
+	return (token.Enabled != nil && *token.Enabled && !token.Expired), nil
+}

rancher2/util.go

@@ -24,6 +24,7 @@ import (
 	"github.com/rancher/norman/types"
 	"golang.org/x/crypto/bcrypt"
 	"gopkg.in/yaml.v2"
+	kubeconfig "k8s.io/client-go/tools/clientcmd/api/v1"
 )
 
 const (
@@ -67,6 +68,82 @@ func IsBase64(s string) bool {
 	return err == nil
 }
 
+func getKubeConfigFromObj(kubeconfig *kubeconfig.Config) (string, error) {
+	if kubeconfig == nil {
+		return "", nil
+	}
+	config, err := interfaceToMap(kubeconfig)
+	if err != nil {
+		return "", err
+	}
+
+	return mapInterfaceToYAML(config)
+}
+
+func getObjFromKubeConfig(config string) (*kubeconfig.Config, error) {
+	kubeconfig := &kubeconfig.Config{}
+	if len(config) == 0 {
+		return kubeconfig, nil
+	}
+	kubeconfigMap, err := ghodssyamlToMapInterface(config)
+	if err != nil {
+		return nil, fmt.Errorf("Yaml unmarshall kube_config %v", err)
+	}
+	kubeconfigJSON, err := mapInterfaceToJSON(kubeconfigMap)
+	if err != nil {
+		return nil, fmt.Errorf("Json marshall kube_config: %v", err)
+	}
+	err = jsonToInterface(kubeconfigJSON, kubeconfig)
+	if err != nil {
+		return nil, fmt.Errorf("Json unmarshall kube_config: %v", err)
+	}
+
+	return kubeconfig, nil
+}
+
+func getTokenFromKubeConfig(config string) (string, error) {
+	if len(config) == 0 {
+		return "", nil
+	}
+	kubeconfig, err := getObjFromKubeConfig(config)
+	if err != nil {
+		return "", err
+	}
+	if kubeconfig == nil || kubeconfig.AuthInfos == nil || len(kubeconfig.AuthInfos) == 0 {
+		return "", nil
+	}
+
+	return kubeconfig.AuthInfos[0].AuthInfo.Token, nil
+}
+
+func getTokenIDFromKubeConfig(config string) (string, error) {
+	token, err := getTokenFromKubeConfig(config)
+	if err != nil {
+		return "", err
+	}
+	return splitTokenID(token), nil
+
+}
+
+func updateKubeConfigToken(config, token string) (string, error) {
+	if len(token) == 0 {
+		return config, nil
+	}
+	if len(config) == 0 {
+		return "", nil
+	}
+	kubeconfig := &kubeconfig.Config{}
+	err := jsonToInterface(config, kubeconfig)
+	if err != nil {
+		return "", err
+	}
+	if kubeconfig == nil || kubeconfig.AuthInfos == nil || len(kubeconfig.AuthInfos) == 0 {
+		return "", nil
+	}
+
+	return kubeconfig.AuthInfos[0].AuthInfo.Token, nil
+}
+
 func TrimSpace(val interface{}) string {
 	return strings.TrimSpace(val.(string))
 }