Disconnect cluster instead of scaling Fleet agent deployment

Taking advantage of k3d, multi-cluster end-to-end tests are able to
simulate disconnection of a cluster by disconnecting the cluster node
container from its Docker network, and reconnecting it.

Disconnecting the server node is enough, as this blocks outbound
traffic, in this case the agent's check-in requests to the controller.
Disconnecting the load balancer node, to block inbound traffic, is not
necessary.

This should represent a more realistic use case than scaling Fleet agent
deployments.

Author: weyfonk

.github/workflows/e2e-multicluster-ci.yml

@@ -83,7 +83,7 @@ jobs:
             -p "443:443@agent:0:direct" \
             --api-port 6443 \
             --agents 1 \
-            --network "nw01"
+            --network "fleet"
       -
         name: Provision k3d Downstream Cluster for agent-initiated registration
         run: |
@@ -92,7 +92,7 @@ jobs:
             -p "444:443@agent:0:direct" \
             --api-port 6644 \
             --agents 1 \
-            --network "nw01"
+            --network "fleet"
       -
         name: Provision k3d Downstream Cluster for manager-initiated registration
         run: |
@@ -101,7 +101,7 @@ jobs:
             -p "445:443@agent:0:direct" \
             --api-port 6645 \
             --agents 1 \
-            --network "nw01"
+            --network "fleet"
       -
         name: Import Images Into k3d
         run: |
@@ -214,6 +214,7 @@ jobs:
         env:
           FLEET_E2E_NS: fleet-local
           FLEET_E2E_NS_DOWNSTREAM: fleet-default
+          FLEET_E2E_CLUSTER_DOWNSTREAM: k3d-downstream
         run: |
           # Force use of non-managed downstream cluster for portability
           export CI_REGISTERED_CLUSTER=$(kubectl get clusters.fleet.cattle.io -n $FLEET_E2E_NS_DOWNSTREAM -o jsonpath='{range .items[*]}{.metadata.name}{"\n"}' | grep -v second)

e2e/multi-cluster/offline_cluster_test.go

@@ -2,6 +2,7 @@ package multicluster_test
 
 import (
 	"encoding/json"
+	"os/exec"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
@@ -18,14 +19,12 @@ import (
 var _ = Describe("Offline cluster detection", func() {
 	var (
 		k     kubectl.Command
-		kd    kubectl.Command
 		asset string
 		name  string
 	)
 
 	BeforeEach(func() {
 		k = env.Kubectl.Context(env.Upstream)
-		kd = env.Kubectl.Context(env.Downstream).Namespace("fleet-default")
 	})
 
 	Context("cluster with deployed workload becomes offline", func() {
@@ -55,8 +54,8 @@ var _ = Describe("Offline cluster detection", func() {
 			Expect(err).ToNot(HaveOccurred())
 
 			DeferCleanup(func() {
-				out, err := kd.Namespace("cattle-fleet-system").Run("scale", "deployment", "fleet-agent", "--replicas=1", "--timeout=60s")
-				Expect(err).ToNot(HaveOccurred(), out)
+				connectCmd := exec.Command("docker", "network", "connect", "fleet", "k3d-"+k3dDownstreamCluster+"-server-0")
+				_, _ = connectCmd.CombinedOutput() // will fail if the node is already connected.
 
 				_, _ = k.Namespace(env.ClusterRegistrationNamespace).Delete("helmop", name)
 			})
@@ -88,8 +87,9 @@ var _ = Describe("Offline cluster detection", func() {
 			}).To(Succeed())
 
 			By("taking the cluster offline")
-			out, err := kd.Namespace("cattle-fleet-system").Run("scale", "deployment", "fleet-agent", "--replicas=0", "--timeout=60s")
-			Expect(err).ToNot(HaveOccurred(), out)
+			disconnectCmd := exec.Command("docker", "network", "disconnect", "fleet", "k3d-"+k3dDownstreamCluster+"-server-0")
+			out, err := disconnectCmd.CombinedOutput()
+			Expect(err).ToNot(HaveOccurred(), string(out))
 
 			By("checking that the bundle deployment and the cluster appear offline")
 			// Cluster should be offline
@@ -117,8 +117,9 @@ var _ = Describe("Offline cluster detection", func() {
 			}).To(Succeed())
 
 			By("taking the cluster back online")
-			out, err = kd.Namespace("cattle-fleet-system").Run("scale", "deployment", "fleet-agent", "--replicas=1", "--timeout=60s")
-			Expect(err).ToNot(HaveOccurred(), out)
+			connectCmd := exec.Command("docker", "network", "connect", "fleet", "k3d-"+k3dDownstreamCluster+"-server-0")
+			out, err = connectCmd.CombinedOutput()
+			Expect(err).ToNot(HaveOccurred(), string(out))
 
 			By("checking the new online state of the cluster")
 			// Cluster should be ready again

e2e/multi-cluster/suite_test.go

@@ -3,6 +3,7 @@ package multicluster_test
 
 import (
 	"os"
+	"strings"
 	"testing"
 
 	"github.com/rancher/fleet/e2e/testenv"
@@ -17,8 +18,9 @@ func TestE2E(t *testing.T) {
 }
 
 var (
-	env       *testenv.Env
-	dsCluster = "second"
+	env                  *testenv.Env
+	dsCluster            = "second" // name of the Fleet Cluster resource
+	k3dDownstreamCluster = "downstream"
 )
 
 var _ = BeforeSuite(func() {
@@ -30,4 +32,8 @@ var _ = BeforeSuite(func() {
 	if dsClusterEnvVar := os.Getenv("CI_REGISTERED_CLUSTER"); dsClusterEnvVar != "" {
 		dsCluster = dsClusterEnvVar
 	}
+
+	if k3dDSClusterEnvVar := os.Getenv("FLEET_E2E_CLUSTER_DOWNSTREAM"); k3dDSClusterEnvVar != "" {
+		k3dDownstreamCluster = strings.TrimPrefix(k3dDSClusterEnvVar, "k3d-")
+	}
 })