hack: Test hybrid cluster upgrades

nodeup/pkg/model/protokube.go

@@ -45,8 +45,10 @@ var _ fi.NodeupModelBuilder = &ProtokubeBuilder{}
 
 // Build is responsible for generating the options for protokube
 func (t *ProtokubeBuilder) Build(c *fi.NodeupModelBuilderContext) error {
-	// check is not a master and we are not using gossip (https://github.com/kubernetes/kops/pull/3091)
-	if !t.IsMaster && !t.UsesLegacyGossip() {
+	// Skip protokube on workers when it isn't needed: either the cluster doesn't use gossip,
+	// or this worker bootstraps via kops-controller NLB (hybrid mode, signalled by APIServerIPs
+	// populated on the boot config). Gossip stays alive on control-plane nodes.
+	if !t.IsMaster && (!t.UsesLegacyGossip() || len(t.BootConfig.APIServerIPs) > 0) {
 		klog.V(2).Infof("skipping the provisioning of protokube on the nodes")
 		return nil
 	}

pkg/apis/kops/cluster.go

@@ -969,6 +969,31 @@ func (c *Cluster) UsesNoneDNS() bool {
 	return false
 }
 
+// UsesLoadBalancerForKopsController returns true when worker nodes reach kops-controller
+// via the cluster load balancer rather than via gossip-populated /etc/hosts. True for all
+// None-DNS clusters, and for gossip clusters on clouds whose model exposes kops-controller
+// on the API load balancer.
+func (c *Cluster) UsesLoadBalancerForKopsController() bool {
+	if c.UsesNoneDNS() {
+		return true
+	}
+	if !c.UsesLegacyGossip() {
+		return false
+	}
+	switch c.GetCloudProvider() {
+	case CloudProviderAWS:
+		// NLB-only: the kops-controller target group requires a Network Load Balancer.
+		lb := c.Spec.API.LoadBalancer
+		return lb != nil && lb.Class == LoadBalancerClassNetwork
+	case CloudProviderHetzner:
+		return true
+	case CloudProviderAzure, CloudProviderDO, CloudProviderGCE, CloudProviderOpenstack, CloudProviderScaleway:
+		return c.Spec.API.LoadBalancer != nil
+	default:
+		return false
+	}
+}
+
 func (c *Cluster) InstallCNIAssets() bool {
 	return c.Spec.Networking.AmazonVPC == nil &&
 		c.Spec.Networking.Calico == nil &&

pkg/model/awsmodel/api_loadbalancer.go

@@ -174,7 +174,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			nlbListeners = append(nlbListeners, listener443)
 		}
 
-		if b.Cluster.UsesNoneDNS() {
+		if b.Cluster.UsesLoadBalancerForKopsController() {
 			{
 				nlbListener := &awstasks.NetworkLoadBalancerListener{
 					Name:                fi.PtrTo(b.NLBListenerName("api", wellknownports.KopsControllerPort)),
@@ -225,9 +225,10 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			Type:              elbv2types.LoadBalancerTypeEnumNetwork,
 		}
 
-		// Wait for all load balancer components to be created (including network interfaces needed for NoneDNS).
-		// Limiting this to clusters using NoneDNS because load balancer creation is quite slow.
-		if b.Cluster.UsesNoneDNS() {
+		// Wait for all load balancer components to be created (including network interfaces needed to
+		// bake NLB ENI IPs into worker nodeup configs). Limiting this to clusters that actually need
+		// those IPs because load balancer creation is quite slow.
+		if b.Cluster.UsesLoadBalancerForKopsController() {
 			nlb.SetWaitForLoadBalancerReady(true)
 		}
 
@@ -264,7 +265,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 			WellKnownServices: []wellknownservices.WellKnownService{wellknownservices.KubeAPIServer},
 		}
 
-		if b.Cluster.UsesNoneDNS() {
+		if b.Cluster.UsesLoadBalancerForKopsController() {
 			lbSpec.CrossZoneLoadBalancing = fi.PtrTo(true)
 		} else if lbSpec.CrossZoneLoadBalancing == nil {
 			lbSpec.CrossZoneLoadBalancing = fi.PtrTo(false)
@@ -341,7 +342,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 				c.AddTask(tg)
 			}
 
-			if b.Cluster.UsesNoneDNS() {
+			if b.Cluster.UsesLoadBalancerForKopsController() {
 				{
 					groupName := b.NLBTargetGroupName("kops-controller")
 					groupTags := b.CloudTags(groupName, false)
@@ -534,7 +535,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 		}
 	}
 
-	if b.Cluster.UsesNoneDNS() {
+	if b.Cluster.UsesLoadBalancerForKopsController() {
 		nodeGroups, err := b.GetSecurityGroups(kops.InstanceGroupRoleNode)
 		if err != nil {
 			return err
@@ -620,7 +621,7 @@ func (b *APILoadBalancerBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 				SourceGroup:   masterGroup.Task,
 				ToPort:        fi.PtrTo(int32(4)),
 			})
-			if b.Cluster.UsesNoneDNS() {
+			if b.Cluster.UsesLoadBalancerForKopsController() {
 				{
 					nlb.WellKnownServices = append(nlb.WellKnownServices, wellknownservices.KopsController)
 					clb.WellKnownServices = append(clb.WellKnownServices, wellknownservices.KopsController)

pkg/model/awsmodel/autoscalinggroup.go

@@ -493,7 +493,7 @@ func (b *AutoscalingGroupModelBuilder) buildAutoScalingGroupTask(c *fi.CloudupMo
 		if b.UseLoadBalancerForAPI() && ig.HasAPIServer() {
 			if b.UseNetworkLoadBalancer() {
 				t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("tcp"))
-				if b.Cluster.UsesNoneDNS() && ig.IsControlPlane() {
+				if b.Cluster.UsesLoadBalancerForKopsController() && ig.IsControlPlane() {
 					t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("kops-controller"))
 					if b.Cluster.Spec.Networking.Cilium != nil && b.Cluster.Spec.Networking.Cilium.EtcdManaged {
 						t.TargetGroups = append(t.TargetGroups, b.LinkToTargetGroup("etcd-cilium"))

pkg/model/azuremodel/api_loadbalancer.go

@@ -104,7 +104,7 @@ func (b *APILoadBalancerModelBuilder) Build(c *fi.CloudupModelBuilderContext) er
 		return fmt.Errorf("unknown load balancer Type: %q", lbSpec.Type)
 	}
 
-	if b.Cluster.UsesLegacyGossip() || b.Cluster.UsesPrivateDNS() || b.Cluster.UsesNoneDNS() {
+	if b.Cluster.UsesLoadBalancerForKopsController() {
 		lb.WellKnownServices = append(lb.WellKnownServices, wellknownservices.KopsController)
 
 		// kops-controller probe: HTTPS on 3988 with /healthz

pkg/model/azuremodel/network.go

@@ -220,7 +220,7 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
 		DestinationApplicationSecurityGroupNames: []*string{fi.PtrTo(b.NameForApplicationSecurityGroupControlPlane())},
 		DestinationPortRange:                     fi.PtrTo("*"),
 	})
-	if b.Cluster.UsesNoneDNS() && b.Cluster.Spec.API.LoadBalancer != nil && b.Cluster.Spec.API.LoadBalancer.Type == kops.LoadBalancerTypePublic {
+	if b.Cluster.UsesLoadBalancerForKopsController() && b.Cluster.Spec.API.LoadBalancer != nil && b.Cluster.Spec.API.LoadBalancer.Type == kops.LoadBalancerTypePublic {
 		// TODO: Limit access to necessary source address prefixes instead of "0.0.0.0/0" and "::/0"
 		nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
 			Name:                                     fi.PtrTo("AllowNodesToKubernetesAPI"),

pkg/model/gcemodel/api_loadbalancer.go

@@ -119,7 +119,7 @@ func (b *APILoadBalancerBuilder) addFirewallRules(c *fi.CloudupModelBuilderConte
 			})
 		}
 
-		if b.Cluster.UsesNoneDNS() {
+		if b.Cluster.UsesLoadBalancerForKopsController() {
 			b.AddFirewallRulesTasks(c, "kops-controller", &gcetasks.FirewallRule{
 				Lifecycle:    b.Lifecycle,
 				Network:      network,
@@ -256,7 +256,7 @@ func (b *APILoadBalancerBuilder) createInternalLB(c *fi.CloudupModelBuilderConte
 				"name":           "api-" + sn.Name,
 			},
 		})
-		if b.Cluster.UsesNoneDNS() {
+		if b.Cluster.UsesLoadBalancerForKopsController() {
 			ipAddress.WellKnownServices = append(ipAddress.WellKnownServices, wellknownservices.KopsController)
 
 			fr := &gcetasks.ForwardingRule{

pkg/model/openstackmodel/servergroup.go

@@ -200,7 +200,7 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.CloudupModelBuilderContex
 		}
 		c.AddTask(portTask)
 
-		if b.Cluster.UsesNoneDNS() && ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
+		if b.Cluster.UsesLoadBalancerForKopsController() && ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
 			portTask.WellKnownServices = append(portTask.WellKnownServices, wellknownservices.KubeAPIServer)
 		}
 

pkg/nodemodel/nodeupconfigbuilder.go

@@ -381,6 +381,11 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, wellKnownAddre
 
 	if cluster.UsesNoneDNS() {
 		bootConfig.APIServerIPs = controlPlaneIPs
+	} else if cluster.UsesLoadBalancerForKopsController() && !ig.HasAPIServer() {
+		// Hybrid mode: worker nodes in gossip clusters reach kops-controller via the
+		// cluster load balancer, so they can resolve kops-controller.internal and
+		// api.internal without protokube. Control-plane nodes keep gossip resolution.
+		bootConfig.APIServerIPs = controlPlaneIPs
 	} else {
 		// If we do have a fixed IP, we use it (on some clouds, initially)
 		// This covers the clouds in UseKopsControllerForNodeConfig which use kops-controller for node config,

tests/e2e/scenarios/upgrade-ab/run-test.sh

@@ -50,6 +50,12 @@ fi
 
 export KOPS_BASE_URL
 
+# Configure a gossip cluster: cluster name must end in .k8s.local. The kubetest2
+# deployer builds the auto-generated cluster name as <jobname>.${KOPS_DNS_DOMAIN}
+# for AWS when --dns=none is not in create-args, so overriding KOPS_DNS_DOMAIN
+# yields a .k8s.local suffix without changing the rest of the name.
+export KOPS_DNS_DOMAIN="tests-kops-aws.k8s.local"
+
 echo "Cleaning up any leaked resources from previous cluster"
 # For KOPS_VERSION_B, the value "latest" means build of the tree
 if [[ "${KOPS_VERSION_B}" == "latest" ]]; then
@@ -93,7 +99,7 @@ ${KUBETEST2} \
     --kubernetes-version="${K8S_VERSION_A}" \
     --control-plane-size="${KOPS_CONTROL_PLANE_COUNT:-1}" \
     --template-path="${KOPS_TEMPLATE:-}" \
-    --create-args="--networking calico ${KOPS_EXTRA_FLAGS:-}"
+    --create-args="--networking cilium --api-loadbalancer-type=public ${KOPS_EXTRA_FLAGS:-}"
 
 # Source the env file to get exported variables, in particular CLUSTER_NAME and KOPS_STATE_STORE
 # shellcheck disable=SC1091

tests/e2e/templates/many-addons.yaml.tmpl

@@ -10,7 +10,7 @@ spec:
   authorization:
     rbac: {}
   awsLoadBalancerController:
-    enabled: true
+    enabled: false
   kubernetesApiAccess:
   - {{.publicIP}}
   certManager:
@@ -54,7 +54,7 @@ spec:
   nodeProblemDetector:
     enabled: true
   nodeTerminationHandler:
-    enabled: true
+    enabled: false
     enableSQSTerminationDraining: true
   nonMasqueradeCIDR: 100.64.0.0/10
   podIdentityWebhook:
@@ -72,7 +72,7 @@ spec:
     maxSurge: "100%"
   topology:
     dns:
-      type: None
+      type: Private
 ---
 
 apiVersion: kops.k8s.io/v1alpha2

upup/models/cloudup/resources/addons/dns-controller.addons.k8s.io/k8s-1.12.yaml.template

@@ -21,7 +21,6 @@ spec:
         k8s-app: dns-controller
         version: {{ KopsVersionForLabel }}
     spec:
-      priorityClassName: system-cluster-critical
       affinity:
         nodeAffinity:
           requiredDuringSchedulingIgnoredDuringExecution:

upup/pkg/fi/cloudup/awstasks/network_load_balancer.go

@@ -370,7 +370,7 @@ func (e *NetworkLoadBalancer) FindAddresses(c *fi.CloudupContext) ([]string, err
 				addresses = append(addresses, fi.ValueOf(lb.LoadBalancer.DNSName))
 			}
 
-			if cluster.UsesNoneDNS() {
+			if cluster.UsesLoadBalancerForKopsController() {
 				nis, err := cloud.FindELBV2NetworkInterfacesByName(fi.ValueOf(e.VPC.ID), aws.ToString(lb.LoadBalancer.LoadBalancerName))
 				if err != nil {
 					return nil, fmt.Errorf("failed to find network interfaces matching %q: %w", aws.ToString(lb.LoadBalancer.LoadBalancerName), err)