Skip to content

Commit 34543b6

Browse files
gflaritygflarity
committed
add kind test for startup comparison
1 parent 6632671 commit 34543b6

File tree

3 files changed

+232
-34
lines changed

3 files changed

+232
-34
lines changed

‎operator/ci/hello_k3s_test.go‎

Lines changed: 204 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@ package ci
1818

1919
import (
2020
"context"
21-
"log"
2221
"testing"
2322

2423
"github.com/k3d-io/k3d/v5/pkg/client"
@@ -30,46 +29,74 @@ import (
3029
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3130
"k8s.io/client-go/kubernetes"
3231
"k8s.io/client-go/tools/clientcmd"
32+
"sigs.k8s.io/kind/pkg/apis/config/v1alpha4"
33+
"sigs.k8s.io/kind/pkg/cluster"
3334
)
3435

35-
func TestWithK3sCluster(t *testing.T) {
36-
ctx := context.Background()
36+
// ClusterConfig holds configuration for creating a k3d cluster
37+
type ClusterConfig struct {
38+
Name string
39+
Servers int
40+
Agents int
41+
Image string
42+
HostPort string
43+
LoadBalancerPort string
44+
}
45+
46+
// DefaultClusterConfig returns a sensible default cluster configuration
47+
func DefaultClusterConfig() ClusterConfig {
48+
return ClusterConfig{
49+
Name: "test-k3d-cluster",
50+
Servers: 1,
51+
Agents: 2,
52+
Image: "rancher/k3s:v1.28.8-k3s1",
53+
HostPort: "6550",
54+
LoadBalancerPort: "8080:80",
55+
}
56+
}
57+
58+
// setupCluster creates a k3d cluster and returns a kubernetes clientset
59+
func setupCluster(ctx context.Context, t *testing.T, cfg ClusterConfig) (*kubernetes.Clientset, *v1alpha5.ClusterConfig, func()) {
60+
t.Logf("📝 Preparing k3d cluster configuration for '%s'...", cfg.Name)
3761

38-
// Define a multi-node cluster configuration.
62+
// Create cluster configuration
3963
clusterConfig := v1alpha5.SimpleConfig{
4064
ObjectMeta: types.ObjectMeta{
41-
Name: "my-k3d-multinode",
65+
Name: cfg.Name,
4266
},
43-
Servers: 1,
44-
Agents: 2,
45-
Image: "rancher/k3s:v1.28.8-k3s1",
46-
// This maps the cluster's internal API server port (6443) to port 6550 on your host machine.
67+
Servers: cfg.Servers,
68+
Agents: cfg.Agents,
69+
Image: cfg.Image,
4770
ExposeAPI: v1alpha5.SimpleExposureOpts{
48-
Host: "0.0.0.0", // Listen on all host network interfaces
49-
HostPort: "6550", // An unused port on your host
71+
Host: "0.0.0.0",
72+
HostPort: cfg.HostPort,
5073
},
5174
Ports: []v1alpha5.PortWithNodeFilters{
5275
{
53-
Port: "8080:80",
76+
Port: cfg.LoadBalancerPort,
5477
NodeFilters: []string{"loadbalancer"},
5578
},
5679
},
5780
}
5881

59-
t.Log("📝 Preparing k3d cluster configuration...")
60-
cfg, err := config.TransformSimpleToClusterConfig(ctx, runtimes.Docker, clusterConfig, "")
82+
// Transform configuration
83+
k3dConfig, err := config.TransformSimpleToClusterConfig(ctx, runtimes.Docker, clusterConfig, "")
6184
if err != nil {
6285
t.Fatalf("Failed to transform config: %v", err)
6386
}
6487

65-
log.Printf("🚀 Creating cluster '%s' with 1 server and 2 agents...", cfg.Cluster.Name)
66-
if err := client.ClusterRun(ctx, runtimes.Docker, cfg); err != nil {
88+
// Create cluster
89+
t.Logf("🚀 Creating cluster '%s' with %d server(s) and %d agent(s)...",
90+
k3dConfig.Cluster.Name, cfg.Servers, cfg.Agents)
91+
92+
if err := client.ClusterRun(ctx, runtimes.Docker, k3dConfig); err != nil {
6793
t.Fatalf("Failed to create cluster: %v", err)
6894
}
6995
t.Log("✅ Cluster created successfully!")
7096

71-
log.Println("📄 Fetching kubeconfig...")
72-
cluster, err := client.ClusterGet(ctx, runtimes.Docker, &cfg.Cluster)
97+
// Get kubeconfig
98+
t.Log("📄 Fetching kubeconfig...")
99+
cluster, err := client.ClusterGet(ctx, runtimes.Docker, &k3dConfig.Cluster)
73100
if err != nil {
74101
t.Fatalf("Could not get cluster: %v", err)
75102
}
@@ -83,39 +110,182 @@ func TestWithK3sCluster(t *testing.T) {
83110
if err != nil {
84111
t.Fatalf("Failed to serialize kubeconfig: %v", err)
85112
}
86-
kubeConfigYaml := string(kubeconfigBytes)
87113

88-
// Now this will work because the server address in the kubeconfig is accessible
89-
config, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeConfigYaml))
114+
// Create kubernetes clientset
115+
restConfig, err := clientcmd.RESTConfigFromKubeConfig(kubeconfigBytes)
90116
if err != nil {
91117
t.Fatalf("Could not create rest config: %v", err)
92118
}
93119

94-
clientset, err := kubernetes.NewForConfig(config)
120+
clientset, err := kubernetes.NewForConfig(restConfig)
95121
if err != nil {
96122
t.Fatalf("Could not create clientset: %v", err)
97123
}
98124

99-
// 5. Run your tests against the cluster!
100-
// For example, list the nodes to prove it's working.
125+
// Return cleanup function
126+
cleanup := func() {
127+
t.Log("🗑️ Deleting cluster...")
128+
if err := client.ClusterDelete(ctx, runtimes.Docker, &k3dConfig.Cluster, k3d.ClusterDeleteOpts{}); err != nil {
129+
t.Logf("Failed to delete cluster: %v", err)
130+
} else {
131+
t.Log("✅ Cluster deleted successfully")
132+
}
133+
}
134+
135+
return clientset, k3dConfig, cleanup
136+
}
137+
138+
// KindClusterConfig holds configuration for creating a kind cluster
139+
type KindClusterConfig struct {
140+
Name string
141+
ControlPlanes int
142+
Workers int
143+
Image string
144+
}
145+
146+
// DefaultKindClusterConfig returns a sensible default kind cluster configuration
147+
func DefaultKindClusterConfig() KindClusterConfig {
148+
return KindClusterConfig{
149+
Name: "test-kind-cluster",
150+
ControlPlanes: 1,
151+
Workers: 2,
152+
Image: "", // Empty means use kind's default
153+
}
154+
}
155+
156+
// setupKindCluster creates a kind cluster and returns a kubernetes clientset
157+
func setupKindCluster(ctx context.Context, t *testing.T, cfg KindClusterConfig) (*kubernetes.Clientset, func()) {
158+
t.Logf("📝 Preparing kind cluster configuration for '%s'...", cfg.Name)
159+
160+
provider := cluster.NewProvider()
161+
162+
// Create cluster configuration
163+
kindConfig := &v1alpha4.Cluster{
164+
Nodes: []v1alpha4.Node{},
165+
}
166+
167+
// Add control plane nodes
168+
for i := 0; i < cfg.ControlPlanes; i++ {
169+
node := v1alpha4.Node{
170+
Role: v1alpha4.ControlPlaneRole,
171+
}
172+
if cfg.Image != "" {
173+
node.Image = cfg.Image
174+
}
175+
kindConfig.Nodes = append(kindConfig.Nodes, node)
176+
}
177+
178+
// Add worker nodes
179+
for i := 0; i < cfg.Workers; i++ {
180+
node := v1alpha4.Node{
181+
Role: v1alpha4.WorkerRole,
182+
}
183+
if cfg.Image != "" {
184+
node.Image = cfg.Image
185+
}
186+
kindConfig.Nodes = append(kindConfig.Nodes, node)
187+
}
188+
189+
// Create cluster
190+
t.Logf("🚀 Creating kind cluster '%s' with %d control-plane(s) and %d worker(s)...",
191+
cfg.Name, cfg.ControlPlanes, cfg.Workers)
192+
193+
if err := provider.Create(
194+
cfg.Name,
195+
cluster.CreateWithV1Alpha4Config(kindConfig),
196+
); err != nil {
197+
t.Fatalf("Failed to create kind cluster: %v", err)
198+
}
199+
t.Log("✅ Kind cluster created successfully!")
200+
201+
// Get kubeconfig
202+
t.Log("📄 Fetching kubeconfig...")
203+
kubeConfigYaml, err := provider.KubeConfig(cfg.Name, false)
204+
if err != nil {
205+
t.Fatalf("Failed to get kubeconfig: %v", err)
206+
}
207+
208+
// Create kubernetes clientset
209+
restConfig, err := clientcmd.RESTConfigFromKubeConfig([]byte(kubeConfigYaml))
210+
if err != nil {
211+
t.Fatalf("Could not create rest config: %v", err)
212+
}
213+
214+
clientset, err := kubernetes.NewForConfig(restConfig)
215+
if err != nil {
216+
t.Fatalf("Could not create clientset: %v", err)
217+
}
218+
219+
// Return cleanup function
220+
cleanup := func() {
221+
t.Log("🗑️ Deleting kind cluster...")
222+
if err := provider.Delete(cfg.Name, ""); err != nil {
223+
t.Logf("Failed to delete kind cluster: %v", err)
224+
} else {
225+
t.Log("✅ Kind cluster deleted successfully")
226+
}
227+
}
228+
229+
return clientset, cleanup
230+
}
231+
232+
func TestWith3dCluster(t *testing.T) {
233+
ctx := context.Background()
234+
235+
// Custom configuration
236+
customCfg := ClusterConfig{
237+
Name: "custom-test-cluster",
238+
Servers: 2,
239+
Agents: 3,
240+
Image: "rancher/k3s:v1.28.8-k3s1",
241+
HostPort: "6551",
242+
LoadBalancerPort: "8081:80",
243+
}
244+
245+
// Setup cluster with custom config
246+
clientset, _, cleanup := setupCluster(ctx, t, customCfg)
247+
defer cleanup()
248+
249+
// Test with custom cluster
101250
nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
102251
if err != nil {
103252
t.Fatalf("could not list nodes: %s", err)
104253
}
105254

106-
t.Logf("✅ Found %d nodes in the cluster\n", len(nodes.Items))
255+
expectedNodes := customCfg.Servers + customCfg.Agents
256+
t.Logf("✅ Found %d nodes in the custom cluster", len(nodes.Items))
257+
258+
if len(nodes.Items) != expectedNodes {
259+
t.Errorf("expected %d nodes, but found %d", expectedNodes, len(nodes.Items))
260+
}
261+
}
262+
263+
// Example of how to use kind with custom configuration
264+
func TestWithKindCluster(t *testing.T) {
265+
ctx := context.Background()
107266

108-
// Verify we have exactly 3 nodes
109-
if len(nodes.Items) != 3 {
110-
t.Errorf("expected 3 nodes, but found %d", len(nodes.Items))
267+
// Custom configuration
268+
customCfg := KindClusterConfig{
269+
Name: "custom-kind-cluster",
270+
ControlPlanes: 2,
271+
Workers: 3,
272+
Image: "kindest/node:v1.28.0", // Optional: specify custom image
111273
}
112274

113-
t.Log("🗑️ Deleting cluster...")
114-
if err := client.ClusterDelete(ctx, runtimes.Docker, &cfg.Cluster, k3d.ClusterDeleteOpts{}); err != nil {
115-
t.Fatalf("Failed to delete cluster: %v", err)
275+
// Setup cluster with custom config
276+
clientset, cleanup := setupKindCluster(ctx, t, customCfg)
277+
defer cleanup()
278+
279+
// Test with custom cluster
280+
nodes, err := clientset.CoreV1().Nodes().List(ctx, metav1.ListOptions{})
281+
if err != nil {
282+
t.Fatalf("could not list nodes: %s", err)
116283
}
117-
t.Log("Cluster deleted.")
118284

119-
// Your integration test logic would go here.
120-
// You can apply manifests, create pods, services, etc.
285+
expectedNodes := customCfg.ControlPlanes + customCfg.Workers
286+
t.Logf("✅ Found %d nodes in the custom kind cluster", len(nodes.Items))
287+
288+
if len(nodes.Items) != expectedNodes {
289+
t.Errorf("expected %d nodes, but found %d", expectedNodes, len(nodes.Items))
290+
}
121291
}

‎operator/go.mod‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,11 +18,14 @@ require (
1818
k8s.io/client-go v0.33.4
1919
k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
2020
sigs.k8s.io/controller-runtime v0.21.0
21+
sigs.k8s.io/kind v0.22.0
2122
)
2223

2324
require (
2425
github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 // indirect
26+
github.com/BurntSushi/toml v1.2.1 // indirect
2527
github.com/Microsoft/go-winio v0.6.2 // indirect
28+
github.com/alessio/shellescape v1.4.1 // indirect
2629
github.com/asaskevich/govalidator v0.0.0-20210307081110-f21760c49a8d // indirect
2730
github.com/beorn7/perks v1.0.1 // indirect
2831
github.com/cenkalti/backoff/v4 v4.3.0 // indirect
@@ -60,6 +63,7 @@ require (
6063
github.com/google/gnostic-models v0.7.0 // indirect
6164
github.com/google/go-cmp v0.7.0 // indirect
6265
github.com/google/go-containerregistry v0.19.1 // indirect
66+
github.com/google/safetext v0.0.0-20220905092116-b49f7bc46da2 // indirect
6367
github.com/gorilla/mux v1.8.0 // indirect
6468
github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
6569
github.com/hashicorp/hcl v1.0.0 // indirect
@@ -72,6 +76,7 @@ require (
7276
github.com/magefile/mage v1.15.0 // indirect
7377
github.com/magiconair/properties v1.8.10 // indirect
7478
github.com/mailru/easyjson v0.9.0 // indirect
79+
github.com/mattn/go-isatty v0.0.20 // indirect
7580
github.com/miekg/pkcs11 v1.1.1 // indirect
7681
github.com/mitchellh/copystructure v1.2.0 // indirect
7782
github.com/mitchellh/go-homedir v1.1.0 // indirect
@@ -91,6 +96,7 @@ require (
9196
github.com/onsi/ginkgo/v2 v2.23.3 // indirect
9297
github.com/opencontainers/go-digest v1.0.0 // indirect
9398
github.com/opencontainers/image-spec v1.1.1 // indirect
99+
github.com/pelletier/go-toml v1.9.4 // indirect
94100
github.com/pelletier/go-toml/v2 v2.1.0 // indirect
95101
github.com/pkg/errors v0.9.1 // indirect
96102
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect

0 commit comments

Comments
 (0)