Skip to content

Commit 9e8ea0d

Browse files
zhzhuang-zjuzhzhuang-zju
committed
init e2e: add e2e tests of command addons and command deinit
Signed-off-by: zhzhuang-zju <[email protected]>
1 parent ffa0dfb commit 9e8ea0d

File tree

3 files changed

+161
-28
lines changed

3 files changed

+161
-28
lines changed

‎test/e2e/framework/deployment.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,3 +304,20 @@ func WaitDeploymentReplicasFitWith(clusters []string, namespace, name string, ex
304304
}, PollTimeout, PollInterval).Should(gomega.Equal(true))
305305
})
306306
}
307+
308+
// WaitDeploymentDisappear wait deployment disappear until timeout.
309+
func WaitDeploymentDisappear(client kubernetes.Interface, namespace, name string) {
310+
klog.Infof("Waiting for deployment(%s/%s) disappears", namespace, name)
311+
gomega.Eventually(func() bool {
312+
_, err := client.AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{})
313+
if err == nil {
314+
return false
315+
}
316+
if apierrors.IsNotFound(err) {
317+
return true
318+
}
319+
320+
klog.Errorf("Failed to get deployment(%s/%s), err: %v", namespace, name, err)
321+
return false
322+
}, PollTimeout, PollInterval).Should(gomega.Equal(true))
323+
}

‎test/e2e/suites/init/base_test.go‎

Lines changed: 136 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ limitations under the License.
1717
package init
1818

1919
import (
20-
"fmt"
2120
"os"
2221
"path/filepath"
2322
"regexp"
@@ -36,6 +35,7 @@ import (
3635
policyv1alpha1 "github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1"
3736
karmada "github.com/karmada-io/karmada/pkg/generated/clientset/versioned"
3837
"github.com/karmada-io/karmada/pkg/util/gclient"
38+
"github.com/karmada-io/karmada/pkg/util/names"
3939
"github.com/karmada-io/karmada/test/e2e/framework"
4040
testhelper "github.com/karmada-io/karmada/test/helper"
4141
)
@@ -128,15 +128,6 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
128128
"--v", "4", pushModeClusterName)
129129
_, err := cmd.ExecOrDie()
130130
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
131-
132-
ginkgo.DeferCleanup(func() {
133-
// Unjoin the push mode cluster
134-
cmd := framework.NewKarmadactlCommand(karmadaConfigFilePath, "", karmadactlPath, "", karmadactlTimeout,
135-
"unjoin", "--cluster-kubeconfig", pushModeKubeConfigPath, "--cluster-context", pushModeClusterName, "--cluster-namespace", "karmada-cluster",
136-
"--v", "4", pushModeClusterName)
137-
_, err := cmd.ExecOrDie()
138-
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
139-
})
140131
})
141132

142133
ginkgo.By("register a pull mode cluster", func() {
@@ -171,18 +162,6 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
171162
)
172163
_, err = cmd.ExecOrDie()
173164
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
174-
175-
ginkgo.DeferCleanup(func() {
176-
ginkgo.By(fmt.Sprintf("Unregistering cluster: %s", pullModeClusterName), func() {
177-
cmd := framework.NewKarmadactlCommand(
178-
kubeconfig, "", karmadactlPath, "", karmadactlTimeout,
179-
"unregister", pullModeClusterName, "--cluster-kubeconfig", pullModeKubeConfigPath,
180-
"--cluster-context", pullModeClusterName, "--namespace", testNamespace,
181-
)
182-
_, err := cmd.ExecOrDie()
183-
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
184-
})
185-
})
186165
})
187166

188167
ginkgo.By("Wait for the new cluster to be ready", func() {
@@ -195,6 +174,65 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
195174
})
196175
})
197176

177+
ginkgo.By("Enable descheduler, metrics-adapter, scheduler-estimator and search by command addons", func() {
178+
// Command "enable all" means enable all addons including descheduler, metrics-adapter, search and scheduler-estimator.
179+
// But each time only one scheduler-estimator can be enabled for one cluster. So, we need to enable
180+
// scheduler-estimator twice for push mode cluster and pull mode cluster respectively.
181+
cmd := framework.NewKarmadactlCommand(
182+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
183+
"addons", "enable", "all",
184+
"--cluster", pullModeClusterName,
185+
"--karmada-kubeconfig", karmadaConfigFilePath,
186+
"--member-kubeconfig", pullModeKubeConfigPath,
187+
"--karmada-descheduler-image", karmadaDeschedulerImage,
188+
"--karmada-metrics-adapter-image", karmadaMetricsAdapterImage,
189+
"--karmada-search-image", karmadaSearchImage,
190+
"--karmada-scheduler-estimator-image", karmadaSchedulerEstimatorImage,
191+
"--member-context", pullModeClusterName,
192+
"--v", "4",
193+
)
194+
_, err := cmd.ExecOrDie()
195+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
196+
197+
cmd = framework.NewKarmadactlCommand(
198+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
199+
"addons", "enable", names.KarmadaSchedulerEstimatorComponentName,
200+
"--cluster", pushModeClusterName,
201+
"--karmada-kubeconfig", karmadaConfigFilePath,
202+
"--member-kubeconfig", pushModeKubeConfigPath,
203+
"--karmada-scheduler-estimator-image", karmadaSchedulerEstimatorImage,
204+
"--member-context", pushModeClusterName,
205+
"--v", "4",
206+
)
207+
_, err = cmd.ExecOrDie()
208+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
209+
})
210+
211+
ginkgo.By("Wait for the addons to be ready", func() {
212+
// Wait for descheduler addon ready
213+
framework.WaitDeploymentFitWith(hostClient, testNamespace, names.KarmadaDeschedulerComponentName, func(deployment *appsv1.Deployment) bool {
214+
return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas
215+
})
216+
217+
// Wait for metrics-adapter addon ready
218+
framework.WaitDeploymentFitWith(hostClient, testNamespace, names.KarmadaMetricsAdapterComponentName, func(deployment *appsv1.Deployment) bool {
219+
return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas
220+
})
221+
222+
// Wait for search addon ready
223+
framework.WaitDeploymentFitWith(hostClient, testNamespace, names.KarmadaSearchComponentName, func(deployment *appsv1.Deployment) bool {
224+
return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas
225+
})
226+
227+
// Wait for scheduler-estimator addon ready
228+
framework.WaitDeploymentFitWith(hostClient, testNamespace, names.GenerateEstimatorDeploymentName(pullModeClusterName), func(deployment *appsv1.Deployment) bool {
229+
return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas
230+
})
231+
framework.WaitDeploymentFitWith(hostClient, testNamespace, names.GenerateEstimatorDeploymentName(pushModeClusterName), func(deployment *appsv1.Deployment) bool {
232+
return deployment.Status.ReadyReplicas == *deployment.Spec.Replicas
233+
})
234+
})
235+
198236
ginkgo.By("Do a simple propagation testing", func() {
199237
deploymentNamespace = "deployment-" + rand.String(RandomStrLength)
200238
framework.CreateNamespace(kubeClient, testhelper.NewNamespace(deploymentNamespace))
@@ -216,13 +254,9 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
216254
})
217255

218256
framework.CreateDeployment(kubeClient, deployment)
219-
ginkgo.DeferCleanup(func() {
220-
framework.RemoveDeployment(kubeClient, deploymentNamespace, deploymentName)
221-
})
257+
defer framework.RemoveDeployment(kubeClient, deploymentNamespace, deploymentName)
222258
framework.CreatePropagationPolicy(karmadaClient, policy)
223-
ginkgo.DeferCleanup(func() {
224-
framework.RemovePropagationPolicy(karmadaClient, deploymentNamespace, policyName)
225-
})
259+
defer framework.RemovePropagationPolicy(karmadaClient, deploymentNamespace, policyName)
226260
framework.WaitDeploymentFitWith(pushModeClusterClient, deployment.Namespace, deployment.Name,
227261
func(*appsv1.Deployment) bool {
228262
return true
@@ -232,6 +266,80 @@ var _ = ginkgo.Describe("Base E2E: deploy a karmada instance by cmd init and do
232266
return true
233267
})
234268
})
269+
270+
ginkgo.By("Disable descheduler, metrics-adapter, scheduler-estimator and search by command addons", func() {
271+
// Command "disable all" means disable all addons including descheduler, metrics-adapter, search and scheduler-estimator.
272+
// But each time only one scheduler-estimator can be disabled for one cluster. So, we need to disable
273+
// scheduler-estimator twice for push mode cluster and pull mode cluster respectively.
274+
cmd := framework.NewKarmadactlCommand(
275+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
276+
"addons", "disable", "all",
277+
"--cluster", pullModeClusterName,
278+
"--karmada-kubeconfig", karmadaConfigFilePath,
279+
"-f", // to skip confirmation prompt
280+
"--v", "4",
281+
)
282+
_, err := cmd.ExecOrDie()
283+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
284+
285+
cmd = framework.NewKarmadactlCommand(
286+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
287+
"addons", "disable", names.KarmadaSchedulerEstimatorComponentName,
288+
"--cluster", pushModeClusterName,
289+
"--karmada-kubeconfig", karmadaConfigFilePath,
290+
"-f", // to skip confirmation prompt
291+
"--v", "4",
292+
)
293+
_, err = cmd.ExecOrDie()
294+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
295+
})
296+
297+
ginkgo.By("Wait for addons to be cleaned up", func() {
298+
// Wait for descheduler addon disappeared
299+
framework.WaitDeploymentDisappear(hostClient, testNamespace, names.KarmadaDeschedulerComponentName)
300+
301+
// Wait for metrics-adapter addon ready
302+
framework.WaitDeploymentDisappear(hostClient, testNamespace, names.KarmadaMetricsAdapterComponentName)
303+
304+
// Wait for search addon ready
305+
framework.WaitDeploymentDisappear(hostClient, testNamespace, names.KarmadaSearchComponentName)
306+
307+
// Wait for scheduler-estimator addon ready
308+
framework.WaitDeploymentDisappear(hostClient, testNamespace, names.GenerateEstimatorDeploymentName(pullModeClusterName))
309+
framework.WaitDeploymentDisappear(hostClient, testNamespace, names.GenerateEstimatorDeploymentName(pushModeClusterName))
310+
})
311+
312+
ginkgo.By("Unjoin member clusters", func() {
313+
cmd := framework.NewKarmadactlCommand(karmadaConfigFilePath, "", karmadactlPath, "", karmadactlTimeout,
314+
"unjoin", "--cluster-kubeconfig", pushModeKubeConfigPath, "--cluster-context", pushModeClusterName, "--cluster-namespace", "karmada-cluster",
315+
"--v", "4", pushModeClusterName)
316+
_, err := cmd.ExecOrDie()
317+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
318+
319+
cmd = framework.NewKarmadactlCommand(
320+
kubeconfig, "", karmadactlPath, "", karmadactlTimeout,
321+
"unregister", pullModeClusterName, "--cluster-kubeconfig", pullModeKubeConfigPath,
322+
"--cluster-context", pullModeClusterName, "--namespace", testNamespace,
323+
)
324+
_, err = cmd.ExecOrDie()
325+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
326+
})
327+
328+
ginkgo.By("Clean up the karmada instance by command deinit", func() {
329+
cmd := framework.NewKarmadactlCommand(
330+
kubeconfig, "", karmadactlPath, testNamespace, karmadactlTimeout,
331+
"deinit", "-f", // to skip confirmation prompt
332+
"--context", hostContext,
333+
"--v", "4",
334+
)
335+
_, err := cmd.ExecOrDie()
336+
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
337+
})
338+
339+
ginkgo.By("Check karmada instance has been cleaned up", func() {
340+
framework.WaitDeploymentDisappear(hostClient, testNamespace, names.KarmadaAPIServerComponentName)
341+
framework.WaitDeploymentDisappear(hostClient, testNamespace, names.KarmadaControllerManagerComponentName)
342+
})
235343
})
236344
})
237345
})

‎test/e2e/suites/init/suite_test.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -76,6 +76,10 @@ var (
7676
karmadaControllerManagerImage string
7777
karmadaSchedulerImage string
7878
karmadaWebhookImage string
79+
karmadaDeschedulerImage string
80+
karmadaMetricsAdapterImage string
81+
karmadaSearchImage string
82+
karmadaSchedulerEstimatorImage string
7983
)
8084

8185
func init() {
@@ -112,6 +116,10 @@ var _ = ginkgo.SynchronizedBeforeSuite(func() []byte { return nil }, func([]byte
112116
karmadaControllerManagerImage = fmt.Sprintf("%s/karmada-controller-manager:%s", registry, version)
113117
karmadaSchedulerImage = fmt.Sprintf("%s/karmada-scheduler:%s", registry, version)
114118
karmadaWebhookImage = fmt.Sprintf("%s/karmada-webhook:%s", registry, version)
119+
karmadaDeschedulerImage = fmt.Sprintf("%s/karmada-descheduler:%s", registry, version)
120+
karmadaMetricsAdapterImage = fmt.Sprintf("%s/karmada-metrics-adapter:%s", registry, version)
121+
karmadaSearchImage = fmt.Sprintf("%s/karmada-search:%s", registry, version)
122+
karmadaSchedulerEstimatorImage = fmt.Sprintf("%s/karmada-scheduler-estimator:%s", registry, version)
115123

116124
goPathCmd := exec.Command("go", "env", "GOPATH")
117125
goPath, err := goPathCmd.CombinedOutput()

0 commit comments

Comments
 (0)