Skip to content

Commit 4880024

Browse files
hawjia.limlimhawjia
hawjia.lim
authored andcommitted
chore(e2e): fix and enable e2e tests
1 parent 51ddb4e commit 4880024

File tree

12 files changed

+157
-62
lines changed

12 files changed

+157
-62
lines changed

.github/workflows/ci.yml

+35
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,41 @@ jobs:
2828

2929
- run: make test
3030

31+
e2e:
32+
runs-on: ubuntu-latest
33+
steps:
34+
- uses: actions/checkout@v3
35+
36+
- name: Set up Go
37+
uses: actions/setup-go@v4
38+
with:
39+
go-version: "1.20"
40+
41+
- name: Install Go dependencies
42+
run: |
43+
go install github.com/onsi/ginkgo/v2/ginkgo
44+
go install sigs.k8s.io/kwok/cmd/{kwok,kwokctl}@v0.3.0
45+
46+
- name: Build cluster
47+
run: CLUSTER_PROVIDER=kwok make dev-up
48+
49+
- name: Build images
50+
run: make images
51+
52+
- name: Run Kubeadmiral
53+
run: |
54+
kwokctl get kubeconfig --name kubeadmiral-host > $HOME/.kube/kubeadmiral/kubeadmiral-host.yaml
55+
docker create \
56+
--network host \
57+
--name kubeadmiral-controller-manager \
58+
ghcr.io/kubewharf/kubeadmiral-controller-manager:latest \
59+
/kubeadmiral-controller-manager --kubeconfig=/etc/kubeconfig --klog-v=4 --cluster-join-timeout=1m
60+
docker cp $HOME/.kube/kubeadmiral/kubeadmiral-host.yaml kubeadmiral-controller-manager:/etc/kubeconfig
61+
docker start kubeadmiral-controller-manager
62+
63+
- name: Run tests
64+
run: KUBECONFIG=$HOME/.kube/kubeadmiral/kubeadmiral-host.yaml EXTRA_GINKGO_FLAGS="-v" make e2e
65+
3166
lint:
3267
runs-on: ubuntu-latest
3368
steps:

test/e2e/automigration/automigration.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ var (
5252
assertNoAutoMigrationDuration = 20 * time.Second
5353
)
5454

55-
var _ = ginkgo.Describe("auto migration", autoMigrationTestLabel, func() {
55+
var _ = ginkgo.Describe("Auto Migration", autoMigrationTestLabel, func() {
5656
f := framework.NewFramework("auto-migration", framework.FrameworkOptions{CreateNamespace: true})
5757

5858
var clusters []*fedcorev1a1.FederatedCluster

test/e2e/federatedcluster/clusterdelete.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -87,9 +87,8 @@ var _ = ginkgo.Describe("Cluster Delete", federatedClusterTestLabels, func() {
8787
// 3. service account info deleted from secret
8888
secret, err = f.HostKubeClient().CoreV1().Secrets(framework.FedSystemNamespace).Get(ctx, secret.Name, metav1.GetOptions{})
8989
gomega.Expect(err).ToNot(gomega.HaveOccurred())
90-
token, ca := getServiceAccountInfo(secret)
90+
token, _ := getServiceAccountInfo(secret)
9191
gomega.Expect(token).To(gomega.BeNil(), "Token data not removed from cluster secret")
92-
gomega.Expect(ca).To(gomega.BeNil(), "Token data not removed from cluster secret")
9392
}
9493

9594
ginkgo.Context("Without cascading delete", func() {

test/e2e/federatedcluster/clusterjoin.go

+1-2
Original file line numberDiff line numberDiff line change
@@ -74,9 +74,8 @@ var _ = ginkgo.Describe("Cluster Join", federatedClusterTestLabels, func() {
7474
ginkgo.By("Assert cluster secret not updated with service account information")
7575
secret, err := f.HostKubeClient().CoreV1().Secrets(framework.FedSystemNamespace).Get(ctx, secret.Name, metav1.GetOptions{})
7676
gomega.Expect(err).ToNot(gomega.HaveOccurred())
77-
token, ca := getServiceAccountInfo(secret)
77+
token, _ := getServiceAccountInfo(secret)
7878
gomega.Expect(token).To(gomega.BeNil())
79-
gomega.Expect(ca).To(gomega.BeNil())
8079
})
8180
})
8281

test/e2e/framework/framework.go

+55-7
Original file line numberDiff line numberDiff line change
@@ -38,11 +38,14 @@ import (
3838
"k8s.io/client-go/dynamic"
3939
kubeclient "k8s.io/client-go/kubernetes"
4040
"k8s.io/client-go/rest"
41+
"k8s.io/client-go/tools/cache"
4142
"k8s.io/client-go/tools/clientcmd"
4243

4344
fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1"
4445
fedclient "github.com/kubewharf/kubeadmiral/pkg/client/clientset/versioned"
46+
fedinformers "github.com/kubewharf/kubeadmiral/pkg/client/informers/externalversions"
4547
"github.com/kubewharf/kubeadmiral/pkg/controllers/common"
48+
"github.com/kubewharf/kubeadmiral/pkg/util/informermanager"
4649
"github.com/kubewharf/kubeadmiral/test/e2e/framework/clusterprovider"
4750
)
4851

@@ -83,6 +86,7 @@ var (
8386
hostFedClient fedclient.Interface
8487
hostDynamicClient dynamic.Interface
8588
hostDiscoveryClient discovery.DiscoveryInterface
89+
ftcManager informermanager.FederatedTypeConfigManager
8690
clusterKubeClients sync.Map
8791
clusterFedClients sync.Map
8892
clusterDynamicClients sync.Map
@@ -93,7 +97,12 @@ func init() {
9397
flag.StringVar(&master, "master", "", "The address of the host Kubernetes cluster.")
9498
flag.StringVar(&kubeconfig, "kubeconfig", "", "The path of the kubeconfig for the host Kubernetes cluster.")
9599
flag.Float64Var(&kubeAPIQPS, "kube-api-qps", 500, "The maximum QPS from each Kubernetes client.")
96-
flag.IntVar(&kubeAPIBurst, "kube-api-burst", 1000, "The maximum burst for throttling requests from each Kubernetes client.")
100+
flag.IntVar(
101+
&kubeAPIBurst,
102+
"kube-api-burst",
103+
1000,
104+
"The maximum burst for throttling requests from each Kubernetes client.",
105+
)
97106

98107
flag.StringVar(&clusterProvider, "cluster-provider", "kwok", "The cluster provider [kwok,kind] to use.")
99108
flag.StringVar(
@@ -102,11 +111,26 @@ func init() {
102111
"kindest/node:v1.20.15@sha256:a32bf55309294120616886b5338f95dd98a2f7231519c7dedcec32ba29699394",
103112
"The node image to use for creating kind test clusters, it should include the image digest.",
104113
)
105-
flag.StringVar(&kwokImagePrefix, "kwok-image-prefix", "registry.k8s.io", "The image prefix used by kwok to pull kubernetes images.")
106-
flag.StringVar(&kwokKubeVersion, "kwok-kube-version", "v1.20.15", "The kubernetes version to be used for kwok member clusters")
114+
flag.StringVar(
115+
&kwokImagePrefix,
116+
"kwok-image-prefix",
117+
"registry.k8s.io",
118+
"The image prefix used by kwok to pull kubernetes images.",
119+
)
120+
flag.StringVar(
121+
&kwokKubeVersion,
122+
"kwok-kube-version",
123+
"v1.20.15",
124+
"The kubernetes version to be used for kwok member clusters",
125+
)
107126

108127
flag.BoolVar(&preserveClusters, "preserve-clusters", false, "If set, clusters created during testing are preserved")
109-
flag.BoolVar(&preserveNamespace, "preserve-namespaces", false, "If set, namespaces created during testing are preserved")
128+
flag.BoolVar(
129+
&preserveNamespace,
130+
"preserve-namespaces",
131+
false,
132+
"If set, namespaces created during testing are preserved",
133+
)
110134
}
111135

112136
var _ = ginkgo.SynchronizedBeforeSuite(
@@ -125,7 +149,7 @@ var _ = ginkgo.SynchronizedBeforeSuite(
125149

126150
return bytes
127151
},
128-
func(data []byte) {
152+
func(ctx context.Context, data []byte) {
129153
params := []string{}
130154
err := json.Unmarshal(data, &params)
131155
gomega.Expect(err).ToNot(gomega.HaveOccurred())
@@ -150,6 +174,21 @@ var _ = ginkgo.SynchronizedBeforeSuite(
150174
hostDiscoveryClient, err = discovery.NewDiscoveryClientForConfig(restConfig)
151175
gomega.Expect(err).ShouldNot(gomega.HaveOccurred())
152176

177+
fedInformerFactory := fedinformers.NewSharedInformerFactory(hostFedClient, 0)
178+
manager := informermanager.NewInformerManager(
179+
hostDynamicClient,
180+
fedInformerFactory.Core().V1alpha1().FederatedTypeConfigs(),
181+
nil,
182+
)
183+
ftcManager = manager
184+
185+
fedInformerFactory.Start(ctx.Done())
186+
manager.Start(ctx)
187+
188+
if !cache.WaitForNamedCacheSync("host-informer-manager", ctx.Done(), ftcManager.HasSynced) {
189+
ginkgo.Fail("failed to wait for host informer manager cache sync")
190+
}
191+
153192
clusterKubeClients = sync.Map{}
154193
clusterFedClients = sync.Map{}
155194
clusterDynamicClients = sync.Map{}
@@ -170,7 +209,9 @@ var _ = ginkgo.SynchronizedBeforeSuite(
170209
defaultClusterWaitTimeout,
171210
)
172211
default:
173-
ginkgo.Fail(fmt.Sprintf("invalid cluster provider, %s or %s accepted", KwokClusterProvider, KindClusterProvider))
212+
ginkgo.Fail(
213+
fmt.Sprintf("invalid cluster provider, %s or %s accepted", KwokClusterProvider, KindClusterProvider),
214+
)
174215
}
175216
},
176217
)
@@ -231,12 +272,19 @@ func (*framework) HostDiscoveryClient() discovery.DiscoveryInterface {
231272
return hostDiscoveryClient
232273
}
233274

275+
func (*framework) FTCManager() informermanager.FederatedTypeConfigManager {
276+
return ftcManager
277+
}
278+
234279
func (f *framework) TestNamespace() *corev1.Namespace {
235280
gomega.Expect(f.namespace).ToNot(gomega.BeNil(), MessageUnexpectedError)
236281
return f.namespace
237282
}
238283

239-
func (f *framework) NewCluster(ctx context.Context, clusterModifiers ...ClusterModifier) (*fedcorev1a1.FederatedCluster, *corev1.Secret) {
284+
func (f *framework) NewCluster(
285+
ctx context.Context,
286+
clusterModifiers ...ClusterModifier,
287+
) (*fedcorev1a1.FederatedCluster, *corev1.Secret) {
240288
clusterName := strings.ToLower(fmt.Sprintf("%s-%s", f.name, rand.String(12)))
241289
cluster, secret := f.clusterProvider.NewCluster(ctx, clusterName)
242290

test/e2e/framework/interface.go

+2
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@ import (
2626

2727
fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1"
2828
fedclient "github.com/kubewharf/kubeadmiral/pkg/client/clientset/versioned"
29+
"github.com/kubewharf/kubeadmiral/pkg/util/informermanager"
2930
)
3031

3132
type FrameworkOptions struct {
@@ -37,6 +38,7 @@ type Framework interface {
3738
HostFedClient() fedclient.Interface
3839
HostDynamicClient() dynamic.Interface
3940
HostDiscoveryClient() discovery.DiscoveryInterface
41+
FTCManager() informermanager.FederatedTypeConfigManager
4042

4143
Name() string
4244
TestNamespace() *corev1.Namespace

test/e2e/framework/policies/propagationpolicy.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,12 @@ func PropagationPolicyForClustersWithPlacements(
3838
},
3939
Spec: fedcorev1a1.PropagationPolicySpec{
4040
SchedulingMode: fedcorev1a1.SchedulingModeDuplicate,
41-
Placements: []fedcorev1a1.ClusterReference{},
41+
Placements: []fedcorev1a1.DesiredPlacement{},
4242
},
4343
}
4444

4545
for _, c := range clusters {
46-
policy.Spec.Placements = append(policy.Spec.Placements, fedcorev1a1.ClusterReference{Cluster: c.Name})
46+
policy.Spec.Placements = append(policy.Spec.Placements, fedcorev1a1.DesiredPlacement{Cluster: c.Name})
4747
}
4848

4949
return policy

test/e2e/resourcepropagation/cronjobs.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -43,7 +43,8 @@ var _ = ginkgo.Describe("CronJob Propagation", func() {
4343
resourcePropagationTest(
4444
f,
4545
&resourcePropagationTestConfig[*batchv1.CronJob]{
46-
gvr: batchv1.SchemeGroupVersion.WithResource("jobs"),
46+
gvr: batchv1.SchemeGroupVersion.WithResource("cronjobs"),
47+
gvk: batchv1.SchemeGroupVersion.WithKind("CronJob"),
4748
objectFactory: resources.GetSimpleV1CronJob,
4849
clientGetter: func(client kubernetes.Interface, namespace string) resourceClient[*batchv1.CronJob] {
4950
return client.BatchV1().CronJobs(namespace)
@@ -56,7 +57,6 @@ var _ = ginkgo.Describe("CronJob Propagation", func() {
5657
return resources.IsV1CronJobScheduledOnce(cronjob), nil
5758
},
5859
statusCollection: &resourceStatusCollectionTestConfig{
59-
gvr: fedtypesv1a1.SchemeGroupVersion.WithResource("federatedcronjobstatuses"),
6060
path: "status",
6161
},
6262
},
@@ -66,7 +66,8 @@ var _ = ginkgo.Describe("CronJob Propagation", func() {
6666
resourcePropagationTest(
6767
f,
6868
&resourcePropagationTestConfig[*batchv1b1.CronJob]{
69-
gvr: batchv1.SchemeGroupVersion.WithResource("jobs"),
69+
gvr: batchv1b1.SchemeGroupVersion.WithResource("cronjobs"),
70+
gvk: batchv1b1.SchemeGroupVersion.WithKind("CronJob"),
7071
objectFactory: resources.GetSimpleV1Beta1CronJob,
7172
clientGetter: func(client kubernetes.Interface, namespace string) resourceClient[*batchv1b1.CronJob] {
7273
return client.BatchV1beta1().CronJobs(namespace)
@@ -79,7 +80,6 @@ var _ = ginkgo.Describe("CronJob Propagation", func() {
7980
return resources.IsV1Beta1CronJobScheduledOnce(cronjob), nil
8081
},
8182
statusCollection: &resourceStatusCollectionTestConfig{
82-
gvr: fedtypesv1a1.SchemeGroupVersion.WithResource("federatedcronjobstatuses"),
8383
path: "status",
8484
},
8585
},

test/e2e/resourcepropagation/deployments.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ var _ = ginkgo.Describe("Deployment Propagation", func() {
3434
f,
3535
&resourcePropagationTestConfig[*appsv1.Deployment]{
3636
gvr: appsv1.SchemeGroupVersion.WithResource("deployments"),
37+
gvk: appsv1.SchemeGroupVersion.WithKind("Deployment"),
3738
objectFactory: resources.GetSimpleDeployment,
3839
clientGetter: func(client kubernetes.Interface, namespace string) resourceClient[*appsv1.Deployment] {
3940
return client.AppsV1().Deployments(namespace)
@@ -46,7 +47,6 @@ var _ = ginkgo.Describe("Deployment Propagation", func() {
4647
return resources.IsDeploymentProgressing(deployment), nil
4748
},
4849
statusCollection: &resourceStatusCollectionTestConfig{
49-
gvr: fedtypesv1a1.SchemeGroupVersion.WithResource("federateddeploymentstatuses"),
5050
path: "status",
5151
},
5252
},

test/e2e/resourcepropagation/framework.go

+26-23
Original file line numberDiff line numberDiff line change
@@ -31,11 +31,12 @@ import (
3131
pkgruntime "k8s.io/apimachinery/pkg/runtime"
3232
"k8s.io/apimachinery/pkg/runtime/schema"
3333
"k8s.io/apimachinery/pkg/types"
34+
jsonutil "k8s.io/apimachinery/pkg/util/json"
3435
"k8s.io/client-go/dynamic"
3536
"k8s.io/client-go/kubernetes"
3637

3738
fedcorev1a1 "github.com/kubewharf/kubeadmiral/pkg/apis/core/v1alpha1"
38-
controllerutil "github.com/kubewharf/kubeadmiral/pkg/controllers/util"
39+
"github.com/kubewharf/kubeadmiral/pkg/util/naming"
3940
"github.com/kubewharf/kubeadmiral/test/e2e/framework"
4041
"github.com/kubewharf/kubeadmiral/test/e2e/framework/policies"
4142
"github.com/kubewharf/kubeadmiral/test/e2e/framework/util"
@@ -71,14 +72,13 @@ type resourceClient[T k8sObject] interface {
7172
}
7273

7374
type resourceStatusCollectionTestConfig struct {
74-
// GVR of the federatedstatus.
75-
gvr schema.GroupVersionResource
7675
// Path to a field in the resource whose value should be collected by status collection.
7776
path string
7877
}
7978

8079
type resourcePropagationTestConfig[T k8sObject] struct {
8180
gvr schema.GroupVersionResource
81+
gvk schema.GroupVersionKind
8282
statusCollection *resourceStatusCollectionTestConfig
8383
// Returns an object template with the given name.
8484
objectFactory func(name string) T
@@ -153,18 +153,17 @@ func resourcePropagationTest[T k8sObject](
153153
})
154154

155155
ginkgo.By("Updating the source object", func() {
156-
patch := []map[string]interface{}{
157-
{
158-
"op": "add",
159-
// escape the / in annotation key
160-
"path": "/metadata/annotations/" + strings.Replace(resourceUpdateTestAnnotationKey, "/", "~1", 1),
161-
"value": resourceUpdateTestAnnotationValue,
156+
patch := map[string]interface{}{
157+
"metadata": map[string]interface{}{
158+
"annotations": map[string]interface{}{
159+
resourceUpdateTestAnnotationKey: resourceUpdateTestAnnotationValue,
160+
},
162161
},
163162
}
164163
patchBytes, err := json.Marshal(patch)
165164
gomega.Expect(err).NotTo(gomega.HaveOccurred(), framework.MessageUnexpectedError)
166165

167-
object, err = hostClient.Patch(ctx, object.GetName(), types.JSONPatchType, patchBytes, metav1.PatchOptions{})
166+
object, err = hostClient.Patch(ctx, object.GetName(), types.StrategicMergePatchType, patchBytes, metav1.PatchOptions{})
168167
gomega.Expect(err).NotTo(gomega.HaveOccurred(), framework.MessageUnexpectedError)
169168
})
170169

@@ -245,29 +244,33 @@ func resourcePropagationTest[T k8sObject](
245244
actualFieldByCluster[cluster.Name] = actualField
246245
}
247246

248-
fedStatusUns, err := f.HostDynamicClient().Resource(config.statusCollection.gvr).Namespace(object.GetNamespace()).Get(
249-
ctx, object.GetName(), metav1.GetOptions{})
247+
ftc, exists := f.FTCManager().GetResourceFTC(config.gvk)
248+
g.Expect(exists).To(gomega.BeTrue())
249+
250+
collectedStatusName := naming.GenerateFederatedObjectName(object.GetName(), ftc.Name)
251+
fedStatus, err := f.HostFedClient().CoreV1alpha1().CollectedStatuses(object.GetNamespace()).Get(
252+
ctx, collectedStatusName, metav1.GetOptions{})
250253
if err != nil && apierrors.IsNotFound(err) {
251254
// status might not have been created yet, use local g to fail only this attempt
252255
g.Expect(err).NotTo(gomega.HaveOccurred(), "Federated status object has not been created")
253256
}
254257
gomega.Expect(err).NotTo(gomega.HaveOccurred(), framework.MessageUnexpectedError)
255258

256-
fedStatus := controllerutil.FederatedResource{}
257-
err = pkgruntime.DefaultUnstructuredConverter.FromUnstructured(fedStatusUns.Object, &fedStatus)
258-
gomega.Expect(err).NotTo(gomega.HaveOccurred(), framework.MessageUnexpectedError)
259-
260-
g.Expect(fedStatus.ClusterStatus).
259+
g.Expect(fedStatus.Clusters).
261260
To(gomega.HaveLen(len(actualFieldByCluster)), "Collected status has wrong number of clusters")
262-
for _, clusterStatus := range fedStatus.ClusterStatus {
263-
actualField, exists := actualFieldByCluster[clusterStatus.ClusterName]
264-
g.Expect(exists).To(gomega.BeTrue(), fmt.Sprintf("collected from unexpected cluster %s", clusterStatus.ClusterName))
265-
266-
collectedField, exists, err := unstructured.NestedFieldNoCopy(clusterStatus.CollectedFields, pathSegments...)
261+
for _, clusterStatus := range fedStatus.Clusters {
262+
actualField, exists := actualFieldByCluster[clusterStatus.Cluster]
263+
g.Expect(exists).
264+
To(gomega.BeTrue(), fmt.Sprintf("collected from unexpected cluster %s", clusterStatus.Cluster))
265+
266+
collectedFields := &map[string]interface{}{}
267+
err := jsonutil.Unmarshal(clusterStatus.CollectedFields.Raw, collectedFields)
268+
gomega.Expect(err).ToNot(gomega.HaveOccurred())
269+
collectedField, exists, err := unstructured.NestedFieldNoCopy(*collectedFields, pathSegments...)
267270
gomega.Expect(err).NotTo(gomega.HaveOccurred(), framework.MessageUnexpectedError)
268271
g.Expect(exists).To(
269272
gomega.BeTrue(),
270-
fmt.Sprintf("collected fields does not contain %q for cluster %s", config.statusCollection.path, clusterStatus.ClusterName),
273+
fmt.Sprintf("collected fields does not contain %q for cluster %s", config.statusCollection.path, clusterStatus.Cluster),
271274
)
272275
g.Expect(collectedField).To(gomega.Equal(actualField), "collected and actual fields differ")
273276
}

0 commit comments

Comments
 (0)