Skip to content

Commit f43a68e

Browse files
Basic intergration and e2e tests
1 parent 37f1093 commit f43a68e

File tree

17 files changed

+426
-208
lines changed

17 files changed

+426
-208
lines changed

.github/workflows/ci.yaml

Lines changed: 44 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
name: Go CI
1+
name: Operator CI
22

33
on:
44
push:
@@ -7,7 +7,7 @@ on:
77
branches: [ main ]
88

99
jobs:
10-
build_and_test:
10+
lint:
1111
runs-on: ubuntu-latest
1212
steps:
1313
- name: Checkout code
@@ -17,16 +17,10 @@ jobs:
1717
uses: actions/setup-go@v5
1818
with:
1919
go-version: '1.24'
20-
20+
2121
- name: Go Tidy
2222
run: go mod tidy && git diff --exit-code
2323

24-
- name: Go Mod
25-
run: go mod download
26-
27-
- name: Go Mod Verify
28-
run: go mod verify
29-
3024
- name: Check controller-gen generated
3125
run: make generate && git diff --exit-code
3226

@@ -38,10 +32,51 @@ jobs:
3832
with:
3933
version: v2.1
4034

35+
build_and_test:
36+
runs-on: ubuntu-latest
37+
steps:
38+
- name: Checkout code
39+
uses: actions/checkout@v4
40+
41+
- name: Set up Go
42+
uses: actions/setup-go@v5
43+
with:
44+
go-version: '1.24'
45+
46+
- name: Go Mod
47+
run: go mod download
48+
49+
- name: Go Mod Verify
50+
run: go mod verify
51+
4152
- name: Build
4253
run: go build -v cmd/main.go
4354

4455
- name: Run tests
4556
run: |
4657
go list ./... | grep -v /e2e
4758
make test-ci
59+
60+
61+
e2e-test:
62+
runs-on: ubuntu-latest
63+
steps:
64+
- name: Checkout code
65+
uses: actions/checkout@v4
66+
67+
- name: Set up Go
68+
uses: actions/setup-go@v5
69+
with:
70+
go-version: '1.24'
71+
72+
- name: Go Mod
73+
run: go mod download
74+
75+
- name: Create k8s Kind Cluster
76+
uses: helm/kind-action@v1
77+
with:
78+
cluster_name: kind
79+
80+
- name: Run e2e tests
81+
run: |
82+
make test-e2e

Tiltfile

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ local_resource(
55
ignore=[
66
'api/*/*generated*',
77
'*/*/*test*',
8+
'*/*/*/*test*',
89
],
910
auto_init=False,
1011
labels=["makefile"],
@@ -17,6 +18,7 @@ docker_build('controller', '.',
1718
'test',
1819
'bin',
1920
'*/*/*test*',
21+
'*/*/*/*test*',
2022
],
2123
)
2224

api/v1alpha1/keepercluster_conditions.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -5,10 +5,10 @@ type KeeperConditionType string
55
const (
66
// KeeperConditionTypeReconcileSucceeded indicates that latest reconciliation was successful.
77
KeeperConditionTypeReconcileSucceeded KeeperConditionType = "TypeReconcileSucceeded"
8-
// KeeperConditionTypeReplicaStartupFailure indicates that certain replicas of the KeeperCluster can not be started.
9-
KeeperConditionTypeReplicaStartupFailure KeeperConditionType = "ReplicaStartupFailure"
10-
// KeeperConditionTypeDegraded indicates that certain replicas of the KeeperCluster are not ready to accept connections.
11-
KeeperConditionTypeDegraded KeeperConditionType = "Degraded"
8+
// KeeperConditionTypeReplicaStartupSucceeded indicates that all replicas of the KeeperCluster are able to start.
9+
KeeperConditionTypeReplicaStartupSucceeded KeeperConditionType = "ReplicaStartupSucceeded"
10+
// KeeperConditionTypeHealthy indicates that all replicas of the KeeperCluster are ready to accept connections.
11+
KeeperConditionTypeHealthy = "Healthy"
1212
// KeeperConditionTypeClusterSizeAligned indicates that KeeperCluster replica amount matches the requested value.
1313
KeeperConditionTypeClusterSizeAligned KeeperConditionType = "ClusterSizeAligned"
1414
// KeeperConditionTypeConfigurationInSync indicates that KeeperCluster configuration is in desired state.

api/v1alpha1/keepercluster_types.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,10 @@ type KeeperClusterSpec struct {
6666
// +optional
6767
Labels map[string]string `json:"labels,omitempty"`
6868

69+
// Additional annotations that are added to resources
70+
// +optional
71+
Annotations map[string]string `json:"annotations,omitempty"`
72+
6973
// PodPolicy used to control resources allocated to pods.
7074
// +optional
7175
PodPolicy PodPolicy `json:"podPolicy,omitempty"`
@@ -111,6 +115,14 @@ func (s *KeeperClusterSpec) WithDefaults() {
111115
Tag: DefaultKeeperContainerTag,
112116
PullPolicy: DefaultKeeperContainerPolicy,
113117
},
118+
Storage: corev1.PersistentVolumeClaimSpec{
119+
AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce},
120+
Resources: corev1.VolumeResourceRequirements{
121+
Requests: corev1.ResourceList{
122+
corev1.ResourceStorage: resource.MustParse("1Gi"),
123+
},
124+
},
125+
},
114126
PodPolicy: PodPolicy{
115127
Resources: corev1.ResourceRequirements{
116128
Requests: corev1.ResourceList{

api/v1alpha1/zz_generated.deepcopy.go

Lines changed: 7 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

config/crd/bases/clickhouse.com_keeperclusters.yaml

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -975,6 +975,11 @@ spec:
975975
x-kubernetes-list-type: atomic
976976
type: object
977977
type: object
978+
annotations:
979+
additionalProperties:
980+
type: string
981+
description: Additional annotations that are added to resources
982+
type: object
978983
image:
979984
description: Container image information
980985
properties:

config/manager/kustomization.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,5 +4,5 @@ apiVersion: kustomize.config.k8s.io/v1beta1
44
kind: Kustomization
55
images:
66
- name: controller
7-
newName: controller
8-
newTag: latest
7+
newName: clickhouse.com/clickhouse-operator
8+
newTag: v0.0.1

config/samples/v1alpha1_keepercluster.yaml

Lines changed: 0 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,12 +7,3 @@ metadata:
77
name: test-keeper-cluster
88
spec:
99
replicas: 3
10-
image:
11-
repository: "docker.io/clickhouse/clickhouse-keeper"
12-
tag: latest
13-
storage:
14-
accessModes:
15-
- ReadWriteOncePod
16-
resources:
17-
requests:
18-
storage: 2Gi

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ require (
1212
github.com/onsi/gomega v1.33.1
1313
github.com/stretchr/testify v1.9.0
1414
go.uber.org/zap v1.26.0
15+
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc
1516
gopkg.in/yaml.v2 v2.4.0
1617
k8s.io/api v0.31.0
1718
k8s.io/apimachinery v0.31.0
@@ -72,7 +73,6 @@ require (
7273
go.opentelemetry.io/otel/trace v1.28.0 // indirect
7374
go.opentelemetry.io/proto/otlp v1.3.1 // indirect
7475
go.uber.org/multierr v1.11.0 // indirect
75-
golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc // indirect
7676
golang.org/x/net v0.38.0 // indirect
7777
golang.org/x/oauth2 v0.21.0 // indirect
7878
golang.org/x/sync v0.12.0 // indirect

internal/controller/keeper/controller_test.go

Lines changed: 103 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -21,67 +21,129 @@ import (
2121

2222
. "github.com/onsi/ginkgo/v2"
2323
. "github.com/onsi/gomega"
24-
"k8s.io/apimachinery/pkg/api/errors"
25-
"k8s.io/apimachinery/pkg/types"
26-
"sigs.k8s.io/controller-runtime/pkg/reconcile"
27-
24+
appsv1 "k8s.io/api/apps/v1"
25+
corev1 "k8s.io/api/core/v1"
2826
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
27+
"k8s.io/apimachinery/pkg/labels"
28+
"k8s.io/apimachinery/pkg/selection"
29+
"k8s.io/utils/ptr"
30+
ctrl "sigs.k8s.io/controller-runtime"
31+
runtime_client "sigs.k8s.io/controller-runtime/pkg/client"
2932

3033
clickhousecomv1alpha1 "github.com/clickhouse-operator/api/v1alpha1"
3134
"github.com/clickhouse-operator/internal/util"
3235
)
3336

3437
var _ = Describe("KeeperCluster Controller", func() {
35-
Context("When reconciling a resource", func() {
36-
const resourceName = "test-resource"
37-
38+
Context("When reconciling standalone KeeperCluster resource", Ordered, func() {
3839
ctx := context.Background()
39-
40-
typeNamespacedName := types.NamespacedName{
41-
Name: resourceName,
42-
Namespace: "default",
40+
cr := &clickhousecomv1alpha1.KeeperCluster{
41+
ObjectMeta: metav1.ObjectMeta{
42+
Name: "standalone",
43+
Namespace: "default",
44+
},
45+
Spec: clickhousecomv1alpha1.KeeperClusterSpec{
46+
Replicas: ptr.To[int32](1),
47+
Labels: map[string]string{
48+
"test-label": "test-val",
49+
},
50+
Annotations: map[string]string{
51+
"test-annotation": "test-val",
52+
},
53+
},
4354
}
44-
keepercluster := &clickhousecomv1alpha1.KeeperCluster{}
45-
46-
BeforeEach(func() {
47-
By("creating the custom resource for the Kind KeeperCluster")
48-
err := k8sClient.Get(ctx, typeNamespacedName, keepercluster)
49-
if err != nil && errors.IsNotFound(err) {
50-
resource := &clickhousecomv1alpha1.KeeperCluster{
51-
ObjectMeta: metav1.ObjectMeta{
52-
Name: resourceName,
53-
Namespace: "default",
54-
},
55-
}
56-
Expect(k8sClient.Create(ctx, resource)).To(Succeed())
57-
}
55+
56+
var services corev1.ServiceList
57+
var configs corev1.ConfigMapList
58+
var statefulsets appsv1.StatefulSetList
59+
60+
It("should create standalone cluster", func() {
61+
By("by creating standalone resource CR")
62+
Expect(k8sClient.Create(ctx, cr)).To(Succeed())
63+
Expect(k8sClient.Get(ctx, cr.GetNamespacedName(), cr)).To(Succeed())
5864
})
5965

60-
AfterEach(func() {
61-
var keepers clickhousecomv1alpha1.KeeperClusterList
62-
Expect(k8sClient.List(ctx, &keepers)).To(Succeed())
66+
It("should successfully create all resources of the new cluster", func() {
67+
By("reconciling the created resource once")
68+
_, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: cr.GetNamespacedName()})
69+
Expect(err).NotTo(HaveOccurred())
70+
Expect(k8sClient.Get(ctx, cr.GetNamespacedName(), cr)).To(Succeed())
6371

64-
for _, keeper := range keepers.Items {
65-
Expect(k8sClient.Delete(ctx, &keeper)).To(Succeed())
72+
appReq, err := labels.NewRequirement(util.LabelAppKey, selection.Equals, []string{cr.SpecificName()})
73+
Expect(err).ToNot(HaveOccurred())
74+
listOpts := &runtime_client.ListOptions{
75+
Namespace: cr.Namespace,
76+
LabelSelector: labels.NewSelector().Add(*appReq),
6677
}
6778

68-
By("Cleanup all keeper clusters")
79+
Expect(k8sClient.List(ctx, &services, listOpts)).To(Succeed())
80+
Expect(services.Items).To(HaveLen(1))
81+
82+
Expect(k8sClient.List(ctx, &configs, listOpts)).To(Succeed())
83+
Expect(configs.Items).To(HaveLen(2))
84+
85+
Expect(k8sClient.List(ctx, &statefulsets, listOpts)).To(Succeed())
86+
Expect(statefulsets.Items).To(HaveLen(1))
6987
})
7088

71-
It("should successfully reconcile the resource", func() {
72-
By("Reconciling the created resource")
73-
controllerReconciler := &ClusterReconciler{
74-
Client: k8sClient,
75-
Scheme: k8sClient.Scheme(),
89+
It("should propagate meta attributes for every resource", func() {
90+
expectedOwnerRef := metav1.OwnerReference{
91+
Kind: "KeeperCluster",
92+
APIVersion: "clickhouse.com/v1alpha1",
93+
UID: cr.UID,
94+
Name: cr.Name,
95+
Controller: ptr.To(true),
96+
BlockOwnerDeletion: ptr.To(true),
97+
}
7698

77-
Reader: k8sClient,
78-
Logger: util.NewZapLogger(logger.Named("keeper")),
99+
By("setting meta attributes for service")
100+
for _, service := range services.Items {
101+
Expect(service.ObjectMeta.OwnerReferences).To(ContainElement(expectedOwnerRef))
102+
for k, v := range cr.Spec.Labels {
103+
Expect(service.ObjectMeta.Labels).To(HaveKeyWithValue(k, v))
104+
}
105+
for k, v := range cr.Spec.Annotations {
106+
Expect(service.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v))
107+
}
108+
}
109+
110+
By("setting meta attributes for configs")
111+
for _, config := range configs.Items {
112+
Expect(config.ObjectMeta.OwnerReferences).To(ContainElement(expectedOwnerRef))
113+
for k, v := range cr.Spec.Labels {
114+
Expect(config.ObjectMeta.Labels).To(HaveKeyWithValue(k, v))
115+
}
116+
for k, v := range cr.Spec.Annotations {
117+
Expect(config.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v))
118+
}
79119
}
80120

81-
_, err := controllerReconciler.Reconcile(ctx, reconcile.Request{
82-
NamespacedName: typeNamespacedName,
83-
})
121+
By("setting meta attributes for statefulsets")
122+
for _, sts := range statefulsets.Items {
123+
Expect(sts.ObjectMeta.OwnerReferences).To(ContainElement(expectedOwnerRef))
124+
for k, v := range cr.Spec.Labels {
125+
Expect(sts.ObjectMeta.Labels).To(HaveKeyWithValue(k, v))
126+
Expect(sts.Spec.Template.ObjectMeta.Labels).To(HaveKeyWithValue(k, v))
127+
}
128+
for k, v := range cr.Spec.Annotations {
129+
Expect(sts.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v))
130+
Expect(sts.Spec.Template.ObjectMeta.Annotations).To(HaveKeyWithValue(k, v))
131+
}
132+
}
133+
})
134+
135+
It("should reflect configuration changes in revisions", func() {
136+
updatedCR := cr.DeepCopy()
137+
updatedCR.Spec.LoggerConfig.LoggerLevel = "warning"
138+
Expect(k8sClient.Update(ctx, updatedCR)).To(Succeed())
139+
_, err := reconciler.Reconcile(ctx, ctrl.Request{NamespacedName: cr.GetNamespacedName()})
84140
Expect(err).NotTo(HaveOccurred())
141+
Expect(k8sClient.Get(ctx, cr.GetNamespacedName(), updatedCR)).To(Succeed())
142+
143+
Expect(updatedCR.Status.ObservedGeneration).To(Equal(updatedCR.Generation))
144+
Expect(updatedCR.Status.UpdateRevision).NotTo(Equal(updatedCR.Status.CurrentRevision))
145+
Expect(updatedCR.Status.ConfigurationRevision).NotTo(Equal(cr.Status.ConfigurationRevision))
146+
Expect(updatedCR.Status.StatefulSetRevision).To(Equal(cr.Status.StatefulSetRevision))
85147
})
86148
})
87149
})

0 commit comments

Comments
 (0)