Skip to content

Commit 1b76288

Browse files
zhzhuang-zjuzhzhuang-zju
committed
init e2e: add e2e tests of command addons and command deinit
Signed-off-by: zhzhuang-zju <[email protected]>
1 parent ffa0dfb commit 1b76288

File tree

3 files changed

+145
-28
lines changed

3 files changed

+145
-28
lines changed

‎test/e2e/framework/deployment.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,3 +304,20 @@ func WaitDeploymentReplicasFitWith(clusters []string, namespace, name string, ex
304304
}, PollTimeout, PollInterval).Should(gomega.Equal(true))
305305
})
306306
}
307+
308+
// WaitDeploymentDisappear waits for a deployment to be removed from a namespace until timeout.
309+
func WaitDeploymentDisappear(client kubernetes.Interface, namespace, name string) {
310+
klog.Infof("Waiting for deployment(%s/%s) disappears", namespace, name)
311+
gomega.Eventually(func() bool {
312+
_, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{})
313+
if err == nil {
314+
return false
315+
}
316+
if apierrors.IsNotFound(err) {
317+
return true
318+
}
319+
320+
klog.Errorf("Failed to get deployment(%s/%s), err: %v", namespace, name, err)
321+
return false
322+
}, PollTimeout, PollInterval).Should(gomega.Equal(true))
323+
}

‎test/e2e/suites/init/base_test.go‎

Lines changed: 120 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ limitations under the License.
1717
package init
1818

1919
import (
20-
"fmt"
2120
"os"
2221
"path/filepath"
2322
"regexp"
@@ -36,6 +35,7 @@ import (
3635
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
3736
karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
3837
"github.com/karmada-io/karmada/pkg/util/gclient"
38+
"github.com/karmada-io/karmada/pkg/util/names"
3939
"github.com/karmada-io/karmada/test/e2e/framework"
4040
testhelper "github.com/karmada-io/karmada/test/helper"
4141
)
@@ -62,6 +62,8 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
6262
var deploymentName string
6363
var policyName string
6464

65+
var addonComponentNames []string
66+
6567
var tempPki string
6668

6769
var etcdDataPath string
@@ -85,6 +87,14 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
8587
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
8688
pullModeClusterClient = kubernetes.NewForConfigOrDie(pullModeClusterRestConfig)
8789

90+
addonComponentNames = []string{
91+
names.KarmadaDeschedulerComponentName,
92+
names.KarmadaMetricsAdapterComponentName,
93+
names.KarmadaSearchComponentName,
94+
names.GenerateEstimatorDeploymentName(pullModeClusterName),
95+
names.GenerateEstimatorDeploymentName(pushModeClusterName),
96+
}
97+
8898
targetClusters = []string{pushModeClusterName, pullModeClusterName}
8999
})
90100

@@ -128,15 +138,6 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
128138
"--v", "4", pushModeClusterName)
129139
_, err := cmd.ExecOrDie()
130140
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
131-
132-
ginkgo.DeferCleanup(func() {
133-
// Unjoin the push mode cluster
134-
cmd := framework.NewKarmadactlCommand(karmadaConfigFilePath, "", karmadactlPath, "", karmadactlTimeout,
135-
"unjoin", "--cluster-kubeconfig", pushModeKubeConfigPath, "--cluster-context", pushModeClusterName, "--cluster-namespace", "karmada-cluster",
136-
"--v", "4", pushModeClusterName)
137-
_, err := cmd.ExecOrDie()
138-
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
139-
})
140141
})
141142

142143
ginkgo.By("register a pull mode cluster", func() {
@@ -171,18 +172,6 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
171172
)
172173
_, err = cmd.ExecOrDie()
173174
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
174-
175-
ginkgo.DeferCleanup(func() {
176-
ginkgo.By(fmt.Sprintf("Unregistering cluster: %s", pullModeClusterName), func() {
177-
cmd := framework.NewKarmadactlCommand(
178-
kubeconfig, "", karmadactlPath, "", karmadactlTimeout,
179-
"unregister", pullModeClusterName, "--cluster-kubeconfig", pullModeKubeConfigPath,
180-
"--cluster-context", pullModeClusterName, "--namespace", testNamespace,
181-
)
182-
_, err := cmd.ExecOrDie()
183-
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
184-
})
185-
})
186175
})
187176

188177
ginkgo.By("Wait for the new cluster to be ready", func() {
@@ -195,6 +184,48 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
195184
})
196185
})
197186

187+
ginkgo.By("Enable descheduler, metrics-adapter, scheduler-estimator and search by command addons", func() {
188+
// Command "enable all" means enable all addons including descheduler, metrics-adapter, search and scheduler-estimator.
189+
// But each time only one scheduler-estimator can be enabled for one cluster. So, we need to enable
190+
// scheduler-estimator twice for push mode cluster and pull mode cluster respectively.
191+
cmd := framework.NewKarmadactlCommand(
192+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
193+
"addons", "enable", "all",
194+
"--cluster", pullModeClusterName,
195+
"--karmada-kubeconfig", karmadaConfigFilePath,
196+
"--member-kubeconfig", pullModeKubeConfigPath,
197+
"--karmada-descheduler-image", karmadaDeschedulerImage,
198+
"--karmada-metrics-adapter-image", karmadaMetricsAdapterImage,
199+
"--karmada-search-image", karmadaSearchImage,
200+
"--karmada-scheduler-estimator-image", karmadaSchedulerEstimatorImage,
201+
"--member-context", pullModeClusterName,
202+
"--v", "4",
203+
)
204+
_, err := cmd.ExecOrDie()
205+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
206+
207+
cmd = framework.NewKarmadactlCommand(
208+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
209+
"addons", "enable", names.KarmadaSchedulerEstimatorComponentName,
210+
"--cluster", pushModeClusterName,
211+
"--karmada-kubeconfig", karmadaConfigFilePath,
212+
"--member-kubeconfig", pushModeKubeConfigPath,
213+
"--karmada-scheduler-estimator-image", karmadaSchedulerEstimatorImage,
214+
"--member-context", pushModeClusterName,
215+
"--v", "4",
216+
)
217+
_, err = cmd.ExecOrDie()
218+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
219+
})
220+
221+
ginkgo.By("Wait for the addons to be ready", func() {
222+
for _, component := range addonComponentNames {
223+
framework.WaitDeploymentFitWith(hostClient, testNamespace, component, func(deployment *appsv1.Deployment) bool {
224+
return framework.CheckDeploymentReadyStatus(deployment, *deployment.Spec.Replicas)
225+
})
226+
}
227+
})
228+
198229
ginkgo.By("Do a simple propagation testing", func() {
199230
deploymentNamespace = "deployment-" + rand.String(RandomStrLength)
200231
framework.CreateNamespace(kubeClient, testhelper.NewNamespace(deploymentNamespace))
@@ -216,13 +247,9 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
216247
})
217248

218249
framework.CreateDeployment(kubeClient, deployment)
219-
ginkgo.DeferCleanup(func() {
220-
framework.RemoveDeployment(kubeClient, deploymentNamespace, deploymentName)
221-
})
250+
defer framework.RemoveDeployment(kubeClient, deploymentNamespace, deploymentName)
222251
framework.CreatePropagationPolicy(karmadaClient, policy)
223-
ginkgo.DeferCleanup(func() {
224-
framework.RemovePropagationPolicy(karmadaClient, deploymentNamespace, policyName)
225-
})
252+
defer framework.RemovePropagationPolicy(karmadaClient, deploymentNamespace, policyName)
226253
framework.WaitDeploymentFitWith(pushModeClusterClient, deployment.Namespace, deployment.Name,
227254
func(*appsv1.Deployment) bool {
228255
return true
@@ -232,6 +259,71 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
232259
return true
233260
})
234261
})
262+
263+
ginkgo.By("Disable descheduler, metrics-adapter, scheduler-estimator and search by command addons", func() {
264+
// Command "disable all" means disable all addons including descheduler, metrics-adapter, search and scheduler-estimator.
265+
// But each time only one scheduler-estimator can be disabled for one cluster. So, we need to disable
266+
// scheduler-estimator twice for push mode cluster and pull mode cluster respectively.
267+
cmd := framework.NewKarmadactlCommand(
268+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
269+
"addons", "disable", "all",
270+
"--cluster", pullModeClusterName,
271+
"--karmada-kubeconfig", karmadaConfigFilePath,
272+
"-f", // to skip confirmation prompt
273+
"--v", "4",
274+
)
275+
_, err := cmd.ExecOrDie()
276+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
277+
278+
cmd = framework.NewKarmadactlCommand(
279+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
280+
"addons", "disable", names.KarmadaSchedulerEstimatorComponentName,
281+
"--cluster", pushModeClusterName,
282+
"--karmada-kubeconfig", karmadaConfigFilePath,
283+
"-f", // to skip confirmation prompt
284+
"--v", "4",
285+
)
286+
_, err = cmd.ExecOrDie()
287+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
288+
})
289+
290+
ginkgo.By("Wait for addons to be cleaned up", func() {
291+
for _, component := range addonComponentNames {
292+
framework.WaitDeploymentDisappear(hostClient, testNamespace, component)
293+
}
294+
})
295+
296+
ginkgo.By("Unjoin member clusters", func() {
297+
cmd := framework.NewKarmadactlCommand(karmadaConfigFilePath, "", karmadactlPath, "", karmadactlTimeout,
298+
"unjoin", "--cluster-kubeconfig", pushModeKubeConfigPath, "--cluster-context", pushModeClusterName, "--cluster-namespace", "karmada-cluster",
299+
"--v", "4", pushModeClusterName)
300+
_, err := cmd.ExecOrDie()
301+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
302+
303+
cmd = framework.NewKarmadactlCommand(
304+
kubeconfig, "", karmadactlPath, "", karmadactlTimeout,
305+
"unregister", pullModeClusterName, "--cluster-kubeconfig", pullModeKubeConfigPath,
306+
"--cluster-context", pullModeClusterName, "--namespace", testNamespace,
307+
)
308+
_, err = cmd.ExecOrDie()
309+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
310+
})
311+
312+
ginkgo.By("Clean up the karmada instance by command deinit", func() {
313+
cmd := framework.NewKarmadactlCommand(
314+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
315+
"deinit", "-f", // to skip confirmation prompt
316+
"--context", hostContext,
317+
"--v", "4",
318+
)
319+
_, err := cmd.ExecOrDie()
320+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
321+
})
322+
323+
ginkgo.By("Check karmada instance has been cleaned up", func() {
324+
framework.WaitDeploymentDisappear(hostClient, testNamespace, names.KarmadaAPIServerComponentName)
325+
framework.WaitDeploymentDisappear(hostClient, testNamespace, names.KarmadaControllerManagerComponentName)
326+
})
235327
})
236328
})
237329
})

‎test/e2e/suites/init/suite_test.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@ var (
7676
karmadaControllerManagerImage string
7777
karmadaSchedulerImage string
7878
karmadaWebhookImage string
79+
karmadaDeschedulerImage string
80+
karmadaMetricsAdapterImage string
81+
karmadaSearchImage string
82+
karmadaSchedulerEstimatorImage string
7983
)
8084

8185
func init() {
@@ -112,6 +116,10 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { return nil }, func([]byte
112116
karmadaControllerManagerImage = fmt.Sprintf("%s/karmada-controller-manager:%s", registry, version)
113117
karmadaSchedulerImage = fmt.Sprintf("%s/karmada-scheduler:%s", registry, version)
114118
karmadaWebhookImage = fmt.Sprintf("%s/karmada-webhook:%s", registry, version)
119+
karmadaDeschedulerImage = fmt.Sprintf("%s/karmada-descheduler:%s", registry, version)
120+
karmadaMetricsAdapterImage = fmt.Sprintf("%s/karmada-metrics-adapter:%s", registry, version)
121+
karmadaSearchImage = fmt.Sprintf("%s/karmada-search:%s", registry, version)
122+
karmadaSchedulerEstimatorImage = fmt.Sprintf("%s/karmada-scheduler-estimator:%s", registry, version)
115123

116124
goPathCmd := exec.Command("go", "env", "GOPATH")
117125
goPath, err := goPathCmd.CombinedOutput()

0 commit comments

Comments
 (0)