Skip to content

Commit 394daf1

Browse files
djboris9djboris9
authored
feat: CI improvements and RBAC fixes
1 parent b99d08d commit 394daf1

File tree

9 files changed

+171
-6
lines changed

9 files changed

+171
-6
lines changed

‎.github/workflows/ci.yml

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,3 +20,60 @@ jobs:
2020
with:
2121
version: latest
2222
args: release --snapshot --rm-dist --skip-publish
23+
- name: Setup kind
24+
uses: engineerd/[email protected]
25+
with:
26+
config: "ci/kind-config.yaml"
27+
- name: Show cluster info and switch to kube-system
28+
run: |
29+
kubectl cluster-info
30+
echo "current-context:" $(kubectl config current-context)
31+
kubectl config set-context --current --namespace kube-system
32+
- name: Deploy ingress-nginx
33+
timeout-minutes: 2
34+
run: |
35+
kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/master/deploy/static/provider/kind/deploy.yaml
36+
kubectl wait --namespace ingress-nginx --for=condition=ready pod --selector=app.kubernetes.io/component=controller --timeout=90s
37+
- name: Import image into kind
38+
run: kind load docker-image postfinance/kubenurse:latest
39+
- name: Deploy example setup
40+
run: |
41+
kubectl apply -k ci/deployment/
42+
- name: Wait for pods
43+
timeout-minutes: 1
44+
run: |
45+
sleep 15 # wait for the scheduler to create pods
46+
kubectl wait pods -l app=kubenurse --for=condition=Ready
47+
kubectl get pods -l app=kubenurse
48+
- name: Check state of pods
49+
timeout-minutes: 2
50+
run: |
51+
sleep 60 # Wait to generate some checks etc.
52+
test $(kubectl get pods -l app=kubenurse | wc -l) -eq 4 # Test for 3 Pods + header
53+
echo "Number of kubenurses is ok"
54+
test $(kubectl logs -l app=kubenurse | grep -v "I'm ready to help you" | wc -l) -eq 0 # Test that there are no unexpected log lines present
55+
echo "Logs are ok"
56+
kubectl get pods -l app=kubenurse -o jsonpath='{range .items[*]}{.status.containerStatuses[0].restartCount}{"\n"}{end}' | (! grep -v 0) # Validate for 0 restarts
57+
echo "Restart count is ok"
58+
curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/metrics | grep "kubenurse_request_" > /dev/null
59+
echo "Metrics contain kubenurse specific data"
60+
curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/metrics | grep "kubenurse_request_" | grep 'type="path_' > /dev/null
61+
echo "Metrics contains neighbours"
62+
curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/metrics | (! grep "kubenurse_errors_total")
63+
echo "Metrics contains no errors"
64+
curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/alive | grep '"neighbourhood_state": "ok"' > /dev/null
65+
echo "Neighbourhood state ok"
66+
curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/alive | grep '"neighbourhood": \[' > /dev/null # If no others are discovered, this is null
67+
echo "Discovery ok: Status page includes neighbours"
68+
- name: Show kubenurse status
69+
run: |
70+
curl -k -s --resolve ingress-nginx-controller.ingress-nginx.svc.cluster.local:443:127.0.0.1 https://ingress-nginx-controller.ingress-nginx.svc.cluster.local:443/alive
71+
if: ${{ always() }}
72+
- name: Describe resources on failure
73+
run: |
74+
kubectl get pods -o wide
75+
kubectl logs -l app=kubenurse
76+
kubectl describe pods -l app=kubenurse
77+
kubectl describe daemonsets -l app=kubenurse
78+
kubectl get events
79+
if: ${{ failure() }}

‎README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,8 @@ Neighbours are discovered by querying the kube-apiserver for every Pod in the
124124
`KUBENURSE_NAMESPACE` with label `KUBENURSE_NEIGHBOUR_FILTER`.
125125
The request is done directly to the Pod-IP and the metric types contains the prefix
126126
`path_` and the hostname of the kubelet on which the neighbour kubenurse should run.
127+
Only kubenurses on nodes that are schedulable are considered as neighbours,
128+
this can be changed by setting `KUBENURSE_ALLOW_UNSCHEDULABLE="true"`.
127129

128130
Metric type: `path_$KUBELET_HOSTNAME`
129131

‎ci/deployment/conf.yaml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
apiVersion: apps/v1
2+
kind: DaemonSet
3+
metadata:
4+
name: kubenurse
5+
spec:
6+
template:
7+
spec:
8+
containers:
9+
- name: kubenurse
10+
env:
11+
- name: KUBENURSE_INSECURE
12+
value: "true"
13+
- name: KUBENURSE_INGRESS_URL
14+
value: https://ingress-nginx-controller.ingress-nginx.svc.cluster.local
15+
---
16+
apiVersion: extensions/v1beta1
17+
kind: Ingress
18+
metadata:
19+
name: kubenurse
20+
spec:
21+
rules:
22+
- host: ingress-nginx-controller.ingress-nginx.svc.cluster.local
23+
http:
24+
paths:
25+
- backend:
26+
serviceName: kubenurse
27+
servicePort: 8080
28+
tls:
29+
- hosts:
30+
- ingress-nginx-controller.ingress-nginx.svc.cluster.local

‎ci/deployment/kustomization.yaml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
bases:
2+
- ../../examples/
3+
patchesStrategicMerge:
4+
- conf.yaml
5+
images:
6+
- name: postfinance/kubenurse
7+
newTag: latest

‎ci/kind-config.yaml

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
kind: Cluster
2+
apiVersion: kind.x-k8s.io/v1alpha4
3+
nodes:
4+
- role: control-plane
5+
kubeadmConfigPatches:
6+
- |
7+
kind: InitConfiguration
8+
nodeRegistration:
9+
kubeletExtraArgs:
10+
node-labels: "ingress-ready=true"
11+
extraPortMappings: # required for ingress-nginx
12+
- containerPort: 80
13+
hostPort: 80
14+
protocol: TCP
15+
- containerPort: 443
16+
hostPort: 443
17+
protocol: TCP
18+
- role: worker
19+
- role: worker

‎examples/daemonset.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ spec:
2020
prometheus.io/scheme: "http"
2121
prometheus.io/scrape: "true"
2222
spec:
23-
serviceAccountName: nurse
23+
serviceAccountName: kubenurse
2424
containers:
2525
- name: kubenurse
2626
env:
@@ -32,7 +32,7 @@ spec:
3232
value: kube-system
3333
- name: KUBENURSE_NEIGHBOUR_FILTER
3434
value: "app=kubenurse"
35-
image: "postfinance/kubenurse:v1.2.0"
35+
image: "postfinance/kubenurse:v1.3.2"
3636
ports:
3737
- containerPort: 8080
3838
protocol: TCP

‎examples/kustomization.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
resources:
2+
- daemonset.yaml
3+
- ingress.yaml
4+
- rbac.yaml
5+
- serviceaccount.yaml
6+
- service.yaml

‎examples/rbac.yaml

Lines changed: 47 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,13 +2,57 @@
22
apiVersion: rbac.authorization.k8s.io/v1
33
kind: RoleBinding
44
metadata:
5-
name: nurse
5+
name: kubenurse
66
namespace: kube-system
7+
roleRef:
8+
apiGroup: rbac.authorization.k8s.io
9+
kind: Role
10+
name: kubenurse
11+
subjects:
12+
- kind: ServiceAccount
13+
name: kubenurse
14+
namespace: kube-system
15+
---
16+
apiVersion: rbac.authorization.k8s.io/v1
17+
kind: Role
18+
metadata:
19+
name: kubenurse
20+
namespace: kube-system
21+
rules:
22+
- apiGroups:
23+
- ""
24+
resources:
25+
- pods
26+
verbs:
27+
- get
28+
- list
29+
- watch
30+
---
31+
# This resource is not needed if KUBENURSE_ALLOW_UNSCHEDULABLE=true
32+
apiVersion: rbac.authorization.k8s.io/v1
33+
kind: ClusterRoleBinding
34+
metadata:
35+
name: kubenurse
736
roleRef:
837
apiGroup: rbac.authorization.k8s.io
938
kind: ClusterRole
10-
name: view
39+
name: kubenurse
1140
subjects:
1241
- kind: ServiceAccount
13-
name: nurse
42+
name: kubenurse
1443
namespace: kube-system
44+
---
45+
# This resource is not needed if KUBENURSE_ALLOW_UNSCHEDULABLE=true
46+
apiVersion: rbac.authorization.k8s.io/v1
47+
kind: ClusterRole
48+
metadata:
49+
name: kubenurse
50+
rules:
51+
- apiGroups:
52+
- ""
53+
resources:
54+
- nodes
55+
verbs:
56+
- list
57+
- get
58+
- watch

‎examples/serviceaccount.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,5 +2,5 @@
22
apiVersion: v1
33
kind: ServiceAccount
44
metadata:
5-
name: nurse
5+
name: kubenurse
66
namespace: kube-system

0 commit comments

Comments
 (0)