Skip to content

Commit cfda6e9

Browse files
armabarmab
authored
Merge pull request #71 from StackStorm/feature/helm-e2e
Helm e2e tests in CircleCI
2 parents 7a05b9b + ec2cc89 commit cfda6e9

File tree

7 files changed

+212
-25
lines changed

7 files changed

+212
-25
lines changed

‎.circleci/config.yml‎

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ version: 2.1
33
# Add additional CircleCI Orbs dependencies
44
orbs:
55
# https://circleci.com/orbs/registry/orb/circleci/kubernetes
6-
kube: circleci/[email protected]
6+
kubernetes: circleci/[email protected]
77
# https://circleci.com/orbs/registry/orb/circleci/helm
88
helm: circleci/[email protected]
99

@@ -60,46 +60,49 @@ jobs:
6060
command: kubeval $(find . -type f)
6161
working_directory: enterprise/stackstorm-ha/templates/
6262

63-
# Spin up lightweight K8s cluster with 'KIND' and run Helm chart & e2e tests on it
63+
# Spin up minikube K8s cluster and run Helm chart & e2e tests on it
6464
helm-e2e:
6565
machine:
6666
# Available images https://circleci.com/docs/2.0/configuration-reference/#available-machine-images
6767
image: ubuntu-1604:201903-01
6868
steps:
6969
- checkout
70-
- kube/install
71-
- helm/install-helm-client
70+
- kubernetes/install
7271
- run:
73-
name: Apply workaround for rabbitmq-ha chart DNS issue
72+
name: Install Minikube
7473
command: |
75-
# busybox image used in rabbitmq-ha chart timeouts on DNS resolution
76-
# multiple 'search' domains in resolv.conf makes it even worse
77-
# https://github.com/kubernetes/kubernetes/issues/66924
78-
sudo sed -i '/^search/ d' /etc/resolv.conf
79-
- run:
80-
name: Install K8s in Docker (KinD)
81-
command: sudo wget -O /usr/local/bin/kind https://github.com/kubernetes-sigs/kind/releases/download/v0.3.0/kind-linux-amd64 && sudo chmod +x /usr/local/bin/kind
74+
sudo curl -fLo /usr/local/bin/minikube https://storage.googleapis.com/minikube/releases/latest/minikube-linux-amd64 && sudo chmod +x /usr/local/bin/minikube
75+
minikube version
8276
- run:
8377
name: Create new K8s cluster
84-
command: |
85-
kind create cluster
86-
echo 'export KUBECONFIG="$(kind get kubeconfig-path)"' >> $BASH_ENV
87-
- helm/install-helm-on-cluster:
88-
enable-cluster-wide-admin-access: true
78+
command: sudo -E minikube start --vm-driver=none --cpus $(nproc) --memory 4096
79+
environment:
80+
CHANGE_MINIKUBE_NONE_USER: true
81+
- helm/install-helm-on-cluster
8982
- run:
90-
name: Update Helm repo dependencies
83+
name: Update stackstorm-ha chart dependencies
9184
command: |
9285
helm repo add incubator https://kubernetes-charts-incubator.storage.googleapis.com/
9386
helm dependency update
9487
- run:
95-
name: Helm install stackstorm-ha chart
96-
command: |
97-
# disable mongodb-ha chart persistent volumes as cluster can't start in CircleCI/KinD env
98-
helm install --debug --wait --name stackstorm-ha . \
99-
--set mongodb-ha.persistentVolume.enabled=false
88+
name: Helm install stackstorm-ha chart (Community)
89+
command: helm install --debug --wait --name stackstorm-ha .
90+
- run:
91+
name: Helm test (Community)
92+
command: helm test stackstorm-ha --parallel --cleanup
93+
- run:
94+
when: always
95+
name: Show created K8s resources (Community)
96+
command: kubectl get all
97+
- run:
98+
name: Helm upgrade stackstorm-ha (Community -> Enterprise)
99+
command: helm upgrade --wait stackstorm-ha . --set enterprise.enabled=true --set enterprise.license=${EWC_LICENSE}
100+
- run:
101+
name: Helm test (Enterprise)
102+
command: helm test stackstorm-ha --parallel
100103
- run:
101104
when: always
102-
name: Show created K8s resources
105+
name: Show created K8s resources (Enterprise)
103106
command: kubectl get all
104107

105108
workflows:
@@ -114,6 +117,24 @@ workflows:
114117
requires:
115118
- helm-lint
116119
- k8s-lint
120+
# Run periodic nightly Helm tests to ensure there are no regressions
121+
helm-nightly:
122+
jobs:
123+
- helm-lint
124+
- k8s-lint:
125+
requires:
126+
- helm-lint
127+
- helm-e2e:
128+
requires:
129+
- helm-lint
130+
- k8s-lint
131+
triggers:
132+
- schedule:
133+
cron: "0 1 * * *"
134+
filters:
135+
branches:
136+
only:
137+
- master
117138

118139
experimental:
119140
notify:

‎CHANGELOG.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,9 @@
22

33
## In Development
44

5+
## v0.17.0
6+
* Add chart e2e `helm test` with BATS. Run CI checks with minikube and CircleCI on every PR/push and nightly.
7+
58
## v0.16.0
69
* st2web now uses HTTP by default (#72). We now recommend you rely on `LoadBalancer` or `Ingress` to add HTTPS layer on top of it.
710

‎Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ apiVersion: v1
22
# Update StackStorm version here to rely on other Docker images tags
33
appVersion: 3.1dev
44
name: stackstorm-ha
5-
version: 0.16.0
5+
version: 0.17.0
66
description: StackStorm K8s Helm Chart, optimized for running StackStorm in HA environment.
77
home: https://stackstorm.com/#product
88
icon: https://avatars1.githubusercontent.com/u/4969009

‎templates/tests/st2tests-configmap.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
apiVersion: v1
2+
kind: ConfigMap
3+
metadata:
4+
name: {{ .Release.Name }}-st2tests
5+
labels:
6+
app: st2tests
7+
tier: tests
8+
vendor: stackstorm
9+
chart: {{ .Chart.Name }}-{{ .Chart.Version }}
10+
release: {{ .Release.Name }}
11+
heritage: {{ .Release.Service }}
12+
data:
13+
st2tests.sh: |
14+
{{ .Files.Get "tests/st2tests.sh" | indent 4 }}

‎templates/tests/st2tests-pod.yaml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
apiVersion: v1
2+
kind: Pod
3+
metadata:
4+
name: "{{ .Release.Name }}-st2tests"
5+
labels:
6+
app: st2tests
7+
tier: tests
8+
vendor: stackstorm
9+
support: {{ template "supportMethod" . }}
10+
chart: {{ .Chart.Name }}-{{ .Chart.Version }}
11+
release: {{ .Release.Name }}
12+
heritage: {{ .Release.Service }}
13+
annotations:
14+
"helm.sh/hook": test-success
15+
spec:
16+
{{- if .Values.enterprise.enabled }}
17+
imagePullSecrets:
18+
- name: {{ .Release.Name }}-st2-license
19+
{{- end }}
20+
initContainers:
21+
# Sidecar container to copy BATS framework to the main container
22+
- name: test-framework
23+
image: dduportal/bats:latest
24+
command:
25+
- 'bash'
26+
- '-ec'
27+
- |
28+
cp -R /opt/bats/ ${BATS_HELPERS_DIR} /tools/
29+
volumeMounts:
30+
- name: tools
31+
mountPath: /tools
32+
# Run the actual BATS tests
33+
containers:
34+
- name: st2tests
35+
image: "{{ template "imageRepository" . }}/st2actionrunner{{ template "enterpriseSuffix" . }}:{{ .Chart.AppVersion }}"
36+
imagePullPolicy: {{ .Values.image.pullPolicy }}
37+
envFrom:
38+
- configMapRef:
39+
name: {{ .Release.Name }}-st2-urls
40+
env:
41+
- name: BATS_HELPERS_DIR
42+
value: /tools/bats-helpers/
43+
- name: ST2_AUTH_USERNAME
44+
valueFrom:
45+
secretKeyRef:
46+
name: {{ .Release.Name }}-st2-auth
47+
key: username
48+
- name: ST2_AUTH_PASSWORD
49+
valueFrom:
50+
secretKeyRef:
51+
name: {{ .Release.Name }}-st2-auth
52+
key: password
53+
- name: ST2_VERSION
54+
value: "{{ .Chart.AppVersion }}"
55+
volumeMounts:
56+
- name: tools
57+
mountPath: /tools
58+
- name: tests
59+
mountPath: /tests
60+
command:
61+
- /tools/bats/libexec/bats
62+
- /tests/st2tests.sh
63+
volumes:
64+
- name: tools
65+
emptyDir: {}
66+
- name: tests
67+
configMap:
68+
name: {{ .Release.Name }}-st2tests
69+
restartPolicy: Never

‎tests/README.md‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
# `stackstorm-ha` Helm chart Tests
2+
3+
This directory contains Helm chart tests, powered by [BATS](https://github.com/sstephenson/bats) Bash Automated Testing System.
4+
Despite the minimum amount of smoke tests written, they ensure that StackStorm was really deployed,
5+
works correctly at its core and alive end-to-end without checking deeply specific functionality or configuration.
6+
If something is terribly wrong, - it'll show up via failed tests.
7+
8+
> Note! As part of the automated CI system, tests are performed on every PR via CircleCI.
9+
> To identify any possible regressions related to upstream Dockerfiles used in chart, nightly CI task was also configured that'll trigger e2e periodically.
10+
11+
To run the tests manually:
12+
```
13+
helm test <release-name>
14+
```
15+
16+
To show the test results:
17+
```
18+
kubectl logs <release-name>-st2tests
19+
```
20+
21+
See https://helm.sh/docs/developing_charts/#chart-tests with more information about Helm chart tests.

‎tests/st2tests.sh‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
#!/usr/bin/env bats
2+
3+
load "${BATS_HELPERS_DIR}/bats-support/load.bash"
4+
load "${BATS_HELPERS_DIR}/bats-assert/load.bash"
5+
load "${BATS_HELPERS_DIR}/bats-file/load.bash"
6+
7+
@test 'st2 version deployed and python env are as expected' {
8+
run st2 --version
9+
assert_success
10+
# st2 3.1dev (7079635), on Python 2.7.12
11+
assert_line --partial "st2 ${ST2_VERSION}"
12+
assert_line --partial 'on Python 2.7.12'
13+
}
14+
15+
@test 'ST2_AUTH_URL service endpoint is accessible and working' {
16+
run curl -v ${ST2_API_URL}
17+
assert_line --partial 'Content-Type: application/json'
18+
assert_line --partial 'St2-Api-Key'
19+
}
20+
21+
@test 'ST2_API_URL service endpoint is accessible and working' {
22+
run curl -v ${ST2_API_URL}
23+
assert_line --partial 'Content-Type: application/json'
24+
assert_line --partial 'St2-Api-Key'
25+
}
26+
27+
@test 'ST2_STREAM_URL service endpoint is accessible and working' {
28+
run curl -v ${ST2_API_URL}
29+
assert_line --partial 'Content-Type: application/json'
30+
assert_line --partial 'St2-Api-Key'
31+
}
32+
33+
@test 'st2 user can log in with auth credentials' {
34+
run st2 login ${ST2_AUTH_USERNAME} --password ${ST2_AUTH_PASSWORD} -w
35+
assert_success
36+
assert_line "Logged in as ${ST2_AUTH_USERNAME}"
37+
assert_file_exist ~/.st2/config
38+
}
39+
40+
@test 'st2 core pack is installed and loaded' {
41+
run st2 action list --pack=core
42+
assert_success
43+
assert_line --partial 'core.local'
44+
}
45+
46+
@test "can execute simple st2 action 'core.local'" {
47+
run st2 run core.local cmd=id
48+
assert_success
49+
assert_line --partial 'return_code: 0'
50+
assert_line --partial "stderr: ''"
51+
assert_line --partial 'stdout: uid=1000(stanley) gid=1000(stanley) groups=1000(stanley)'
52+
assert_line --partial 'succeeded: true'
53+
}
54+
55+
@test 'st2 chatops core rule is loaded' {
56+
run st2 rule list
57+
assert_success
58+
assert_line --partial 'chatops.notify'
59+
}

0 commit comments

Comments
 (0)