Skip to content

Commit 1fda3dd

Browse files
sfotonysfotony
authored
Update local testing to use the helm chart, add kubernetes service creation to chart (adobe#349)
* Update local-test* make targets to use the helm chart, add service option to chart * make format
1 parent b18ec49 commit 1fda3dd

File tree

9 files changed

+125
-35
lines changed

9 files changed

+125
-35
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
# build artifacts
66
kubeconfig*
77
dist
8-
k8s-shredder
8+
/k8s-shredder
99
my-k8s-shredder-values.yaml
1010

1111
# Test binary, build with `go test -c`

‎Makefile‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ build: check-license lint vet security unit-test ## Builds the local Docker cont
113113
local-test: build ## Test docker image in a kind cluster (with Karpenter drift and node label detection disabled)
114114
@hash kind 2>/dev/null && { \
115115
echo "Test docker image in a kind cluster..."; \
116-
./internal/testing/local_env_prep.sh "${K8S_SHREDDER_VERSION}" "${KINDNODE_VERSION}" "${TEST_CLUSTERNAME}" "${KUBECONFIG_LOCALTEST}" && \
116+
./internal/testing/local_env_prep_helm.sh "${K8S_SHREDDER_VERSION}" "${KINDNODE_VERSION}" "${TEST_CLUSTERNAME}" "${KUBECONFIG_LOCALTEST}" && \
117117
./internal/testing/cluster_upgrade.sh "${TEST_CLUSTERNAME}" "${KUBECONFIG_LOCALTEST}" || \
118118
exit 1; \
119119
} || { \
@@ -123,7 +123,7 @@ local-test: build ## Test docker image in a kind cluster (with Karpenter drift a
123123
local-test-karpenter: build ## Test docker image in a kind cluster with Karpenter and drift detection enabled
124124
@hash kind 2>/dev/null && { \
125125
echo "Test docker image in a kind cluster with Karpenter..."; \
126-
./internal/testing/local_env_prep_karpenter.sh "${K8S_SHREDDER_VERSION}" "${KINDNODE_VERSION}" "${TEST_CLUSTERNAME_KARPENTER}" "${KUBECONFIG_KARPENTER}" && \
126+
./internal/testing/local_env_prep_karpenter_helm.sh "${K8S_SHREDDER_VERSION}" "${KINDNODE_VERSION}" "${TEST_CLUSTERNAME_KARPENTER}" "${KUBECONFIG_KARPENTER}" && \
127127
./internal/testing/cluster_upgrade_karpenter.sh "${TEST_CLUSTERNAME_KARPENTER}" "${KUBECONFIG_KARPENTER}" || \
128128
exit 1; \
129129
} || { \
@@ -133,7 +133,7 @@ local-test-karpenter: build ## Test docker image in a kind cluster with Karpente
133133
local-test-node-labels: build ## Test docker image in a kind cluster with node label detection enabled
134134
@hash kind 2>/dev/null && { \
135135
echo "Test docker image in a kind cluster with node label detection..."; \
136-
./internal/testing/local_env_prep_node_labels.sh "${K8S_SHREDDER_VERSION}" "${KINDNODE_VERSION}" "${TEST_CLUSTERNAME_NODE_LABELS}" "${KUBECONFIG_NODE_LABELS}" && \
136+
./internal/testing/local_env_prep_node_labels_helm.sh "${K8S_SHREDDER_VERSION}" "${KINDNODE_VERSION}" "${TEST_CLUSTERNAME_NODE_LABELS}" "${KUBECONFIG_NODE_LABELS}" && \
137137
./internal/testing/cluster_upgrade_node_labels.sh "${TEST_CLUSTERNAME_NODE_LABELS}" "${KUBECONFIG_NODE_LABELS}" || \
138138
exit 1; \
139139
} || { \
@@ -163,7 +163,7 @@ e2e-tests: ## Run e2e tests for k8s-shredder deployed in a local kind cluster
163163
.PHONY: demo.prep demo.run demo.rollback
164164
demo.prep: build ## Setup demo cluster
165165
echo "Setup demo cluster..."
166-
./internal/testing/local_env_prep.sh "${K8S_SHREDDER_VERSION}" "${KINDNODE_VERSION}" "${TEST_CLUSTERNAME}"
166+
./internal/testing/local_env_prep_helm.sh "${K8S_SHREDDER_VERSION}" "${KINDNODE_VERSION}" "${TEST_CLUSTERNAME}"
167167

168168
demo.run: ## Run demo
169169
./internal/testing/cluster_upgrade.sh "${TEST_CLUSTERNAME}"

‎charts/k8s-shredder/Chart.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,5 +12,5 @@ maintainers:
1212
- name: sfotony
1313
email: [email protected]
1414
url: https://adobe.com
15-
version: 0.2.3
15+
version: 0.2.4
1616
appVersion: v0.3.1

‎charts/k8s-shredder/README.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
# k8s-shredder
22

3-
![Version: 0.2.3](https://img.shields.io/badge/Version-0.2.3-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.3.1](https://img.shields.io/badge/AppVersion-v0.3.1-informational?style=flat-square)
3+
![Version: 0.2.4](https://img.shields.io/badge/Version-0.2.4-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: v0.3.1](https://img.shields.io/badge/AppVersion-v0.3.1-informational?style=flat-square)
44

55
a novel way of dealing with kubernetes nodes blocked from draining
66

@@ -52,6 +52,13 @@ a novel way of dealing with kubernetes nodes blocked from draining
5252
| resources.requests.cpu | string | `"250m"` | CPU cores requested for the container (guaranteed allocation) |
5353
| resources.requests.memory | string | `"250Mi"` | Memory requested for the container (guaranteed allocation) |
5454
| securityContext | object | `{}` | Security context applied to the k8s-shredder container |
55+
| service | object | `{"annotations":{},"create":false,"labels":{},"port":8080,"targetPort":"metrics","type":"ClusterIP"}` | Kubernetes service configuration |
56+
| service.annotations | object | `{}` | Additional annotations for the service |
57+
| service.create | bool | `false` | Create a service for k8s-shredder metrics endpoint |
58+
| service.labels | object | `{}` | Additional labels for the service |
59+
| service.port | int | `8080` | Service port for metrics endpoint |
60+
| service.targetPort | string | `"metrics"` | Target port for metrics endpoint |
61+
| service.type | string | `"ClusterIP"` | Service type (ClusterIP, NodePort, LoadBalancer) |
5562
| serviceAccount | object | `{"annotations":{},"create":true,"name":"k8s-shredder"}` | Kubernetes service account configuration |
5663
| serviceAccount.annotations | object | `{}` | Additional annotations for the service account (useful for IAM roles, etc.) |
5764
| serviceAccount.create | bool | `true` | Create a service account for k8s-shredder |

‎charts/k8s-shredder/templates/service.yaml‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
{{- if .Values.service.create }}
2+
apiVersion: v1
3+
kind: Service
4+
metadata:
5+
name: {{ include "k8s-shredder.fullname" . }}
6+
namespace: {{ .Release.Namespace }}
7+
labels:
8+
{{ include "k8s-shredder.labels" . | indent 4 }}
9+
{{- if .Values.service.labels }}
10+
{{ toYaml .Values.service.labels | indent 4 }}
11+
{{- end }}
12+
{{- if .Values.service.annotations }}
13+
annotations:
14+
{{ toYaml .Values.service.annotations | indent 4 }}
15+
{{- end }}
16+
spec:
17+
type: {{ .Values.service.type }}
18+
selector:
19+
{{ include "k8s-shredder.matchLabels" . | indent 4 }}
20+
ports:
21+
- name: metrics
22+
port: {{ .Values.service.port }}
23+
targetPort: {{ .Values.service.targetPort }}
24+
protocol: TCP
25+
{{- end }}

‎charts/k8s-shredder/values.yaml‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,20 @@ serviceAccount:
7070
name: k8s-shredder
7171
# -- Additional annotations for the service account (useful for IAM roles, etc.)
7272
annotations: {}
73+
# -- Kubernetes service configuration
74+
service:
75+
# -- Create a service for k8s-shredder metrics endpoint
76+
create: false
77+
# -- Service type (ClusterIP, NodePort, LoadBalancer)
78+
type: ClusterIP
79+
# -- Service port for metrics endpoint
80+
port: 8080
81+
# -- Target port for metrics endpoint
82+
targetPort: metrics
83+
# -- Additional annotations for the service
84+
annotations: {}
85+
# -- Additional labels for the service
86+
labels: {}
7387
# -- Annotations to add to k8s-shredder pod(s)
7488
podAnnotations: {}
7589
# -- Additional labels to add to k8s-shredder pod(s)

‎internal/testing/local_env_prep.sh‎ renamed to ‎internal/testing/local_env_prep_helm.sh‎

Lines changed: 23 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,6 @@ else
3333
kubectl create namespace ns-team-k8s-shredder-test
3434
fi
3535

36-
echo "KIND: setting up k8s-shredder rbac..."
37-
kubectl apply -f "${test_dir}/rbac.yaml"
38-
3936
if [[ ${ENABLE_APISERVER_DEBUG} == "true" ]]
4037
then
4138
echo -e "K8S_SHREDDER: Enable debug logging on apiserver"
@@ -45,8 +42,25 @@ then
4542
curl -s -X PUT -d '5' "$APISERVER"/debug/flags/v --header "Authorization: Bearer $TOKEN" -k
4643
fi
4744

48-
echo "KIND: deploying k8s-shredder..."
49-
kubectl apply -f "${test_dir}/k8s-shredder.yaml"
45+
echo "KIND: deploying k8s-shredder using Helm chart..."
46+
# Use Helm to deploy k8s-shredder with test-specific configuration
47+
helm install k8s-shredder "${test_dir}/../../charts/k8s-shredder" \
48+
--namespace kube-system \
49+
--set image.registry=adobe/k8s-shredder \
50+
--set image.tag="${K8S_SHREDDER_VERSION}" \
51+
--set image.pullPolicy=Never \
52+
--set shredder.EvictionLoopInterval=10s \
53+
--set shredder.ParkedNodeTTL=30s \
54+
--set shredder.RollingRestartThreshold=0.5 \
55+
--set shredder.EnableKarpenterDriftDetection=false \
56+
--set shredder.EnableNodeLabelDetection=false \
57+
--set logLevel=info \
58+
--set logFormat=text \
59+
--set dryRun=false \
60+
--set service.create=true \
61+
--set service.type=ClusterIP \
62+
--set service.port=8080 \
63+
--set service.targetPort=metrics
5064

5165
echo "KIND: deploying prometheus..."
5266
kubectl apply -f "${test_dir}/prometheus_stuffs.yaml"
@@ -61,7 +75,6 @@ kubectl apply -f "${test_dir}/test_apps.yaml"
6175
rollout_uid=$(kubectl -n ns-team-k8s-shredder-test get rollout test-app-argo-rollout -o jsonpath='{.metadata.uid}')
6276
sed "s/REPLACE_WITH_ROLLOUT_UID/${rollout_uid}/" < "${test_dir}/test_apps.yaml" | kubectl apply -f -
6377

64-
6578
echo "K8S_SHREDDER: waiting for k8s-shredder deployment to become ready!"
6679
retry_count=0
6780
i=1
@@ -71,7 +84,7 @@ while [[ ${status} == *"False"* || -z ${status} ]]; do
7184
if [[ ${retry_count} == 600 ]]; then echo "Timeout exceeded!" && exit 1; fi
7285
# shellcheck disable=SC2059
7386
printf "\b${sp:i++%${#sp}:1}" && sleep 0.5;
74-
status=$(kubectl get pods -n kube-system -l app=k8s-shredder -o json | \
87+
status=$(kubectl get pods -n kube-system -l app.kubernetes.io/name=k8s-shredder -o json | \
7588
jq '.items[].status.conditions[] | select(.type=="Ready")| .status' 2> /dev/null)
7689
retry_count=$((retry_count+1))
7790
done
@@ -89,7 +102,7 @@ while [[ $(kubectl get pdb -n ns-team-k8s-shredder-test test-app-argo-rollout \
89102
done
90103

91104
echo ""
92-
kubectl logs -l app=k8s-shredder -n kube-system
105+
kubectl logs -l app.kubernetes.io/name=k8s-shredder -n kube-system
93106

94107
echo "K8S_SHREDDER: waiting for prometheus deployment to become ready!"
95108
retry_count=0
@@ -109,7 +122,7 @@ kubectl port-forward -n kube-system svc/k8s-shredder --kubeconfig=${KUBECONFIG_F
109122
It can take few minutes before seeing k8s-shredder metrics..."
110123

111124
echo -e "K8S_SHREDDER: You can access k8s-shredder logs by running
112-
kubectl logs -n kube-system -l app=k8s-shredder --kubeconfig=${KUBECONFIG_FILE} \n"
125+
kubectl logs -n kube-system -l app.kubernetes.io/name=k8s-shredder --kubeconfig=${KUBECONFIG_FILE} \n"
113126

114127
echo -e "K8S_SHREDDER: You can access prometheus metrics at http://localhost:1234 after running
115-
kubectl port-forward -n kube-system svc/prometheus --kubeconfig=${KUBECONFIG_FILE} 1234:9090\n"
128+
kubectl port-forward -n kube-system svc/prometheus --kubeconfig=${KUBECONFIG_FILE} 1234:9090\n"

‎internal/testing/local_env_prep_karpenter.sh‎ renamed to ‎internal/testing/local_env_prep_karpenter_helm.sh‎

Lines changed: 23 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,6 @@ else
3333
kubectl create namespace ns-team-k8s-shredder-test
3434
fi
3535

36-
echo "KIND: setting up k8s-shredder rbac..."
37-
kubectl apply -f "${test_dir}/rbac.yaml"
38-
3936
if [[ ${ENABLE_APISERVER_DEBUG} == "true" ]]
4037
then
4138
echo -e "K8S_SHREDDER: Enable debug logging on apiserver"
@@ -119,8 +116,25 @@ echo "KARPENTER: Mock Karpenter CRDs created for testing purposes"
119116
echo "KARPENTER: Creating mock NodeClaims for testing k8s-shredder drift detection..."
120117
# We'll create mock NodeClaims in the cluster upgrade script
121118

122-
echo "KIND: deploying k8s-shredder..."
123-
kubectl apply -f "${test_dir}/k8s-shredder-karpenter.yaml"
119+
echo "KIND: deploying k8s-shredder using Helm chart with Karpenter drift detection..."
120+
# Use Helm to deploy k8s-shredder with Karpenter drift detection enabled
121+
helm install k8s-shredder "${test_dir}/../../charts/k8s-shredder" \
122+
--namespace kube-system \
123+
--set image.registry=adobe/k8s-shredder \
124+
--set image.tag="${K8S_SHREDDER_VERSION}" \
125+
--set image.pullPolicy=Never \
126+
--set shredder.EvictionLoopInterval=30s \
127+
--set shredder.ParkedNodeTTL=2m \
128+
--set shredder.RollingRestartThreshold=0.5 \
129+
--set shredder.EnableKarpenterDriftDetection=true \
130+
--set shredder.EnableNodeLabelDetection=false \
131+
--set logLevel=debug \
132+
--set logFormat=text \
133+
--set dryRun=false \
134+
--set service.create=true \
135+
--set service.type=ClusterIP \
136+
--set service.port=8080 \
137+
--set service.targetPort=metrics
124138

125139
echo "KIND: deploying prometheus..."
126140
kubectl apply -f "${test_dir}/prometheus_stuffs_karpenter.yaml"
@@ -146,7 +160,7 @@ while [[ ${status} == *"False"* || -z ${status} ]]; do
146160
if [[ ${retry_count} == 600 ]]; then echo "Timeout exceeded!" && exit 1; fi
147161
# shellcheck disable=SC2059
148162
printf "\b${sp:i++%${#sp}:1}" && sleep 0.5;
149-
status=$(kubectl get pods -n kube-system -l app=k8s-shredder -o json | \
163+
status=$(kubectl get pods -n kube-system -l app.kubernetes.io/name=k8s-shredder -o json | \
150164
jq '.items[].status.conditions[] | select(.type=="Ready")| .status' 2> /dev/null)
151165
retry_count=$((retry_count+1))
152166
done
@@ -164,7 +178,7 @@ while [[ $(kubectl get pdb -n ns-team-k8s-shredder-test test-app-argo-rollout \
164178
done
165179

166180
echo ""
167-
kubectl logs -l app=k8s-shredder -n kube-system
181+
kubectl logs -l app.kubernetes.io/name=k8s-shredder -n kube-system
168182

169183
echo "K8S_SHREDDER: waiting for prometheus deployment to become ready!"
170184
retry_count=0
@@ -184,12 +198,12 @@ kubectl port-forward -n kube-system svc/k8s-shredder --kubeconfig=${KUBECONFIG_F
184198
It can take few minutes before seeing k8s-shredder metrics..."
185199

186200
echo -e "K8S_SHREDDER: You can access k8s-shredder logs by running
187-
kubectl logs -n kube-system -l app=k8s-shredder --kubeconfig=${KUBECONFIG_FILE} \n"
201+
kubectl logs -n kube-system -l app.kubernetes.io/name=k8s-shredder --kubeconfig=${KUBECONFIG_FILE} \n"
188202

189203
echo -e "K8S_SHREDDER: You can access prometheus metrics at http://localhost:1234 after running
190204
kubectl port-forward -n kube-system svc/prometheus --kubeconfig=${KUBECONFIG_FILE} 1234:9090\n"
191205

192206
echo "KARPENTER: Environment setup complete!"
193207
echo "KARPENTER: Mock Karpenter CRDs are ready for testing"
194208
echo ""
195-
echo "KARPENTER: To test drift detection, the upgrade script will create mock drifted NodeClaims..."
209+
echo "KARPENTER: To test drift detection, the upgrade script will create mock drifted NodeClaims..."

‎internal/testing/local_env_prep_node_labels.sh‎ renamed to ‎internal/testing/local_env_prep_node_labels_helm.sh‎

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,9 +33,6 @@ else
3333
kubectl create namespace ns-team-k8s-shredder-test
3434
fi
3535

36-
echo "KIND: setting up k8s-shredder rbac..."
37-
kubectl apply -f "${test_dir}/rbac.yaml"
38-
3936
if [[ ${ENABLE_APISERVER_DEBUG} == "true" ]]
4037
then
4138
echo -e "K8S_SHREDDER: Enable debug logging on apiserver"
@@ -48,8 +45,28 @@ fi
4845
echo "NODE_LABELS: This test environment will demonstrate node label detection functionality"
4946
echo "NODE_LABELS: k8s-shredder will detect nodes with specific labels and park them"
5047

51-
echo "KIND: deploying k8s-shredder with node label detection enabled..."
52-
kubectl apply -f "${test_dir}/k8s-shredder-node-labels.yaml"
48+
echo "KIND: deploying k8s-shredder using Helm chart with node label detection enabled..."
49+
# Use Helm to deploy k8s-shredder with node label detection enabled
50+
helm install k8s-shredder "${test_dir}/../../charts/k8s-shredder" \
51+
--namespace kube-system \
52+
--set image.registry=adobe/k8s-shredder \
53+
--set image.tag="${K8S_SHREDDER_VERSION}" \
54+
--set image.pullPolicy=Never \
55+
--set shredder.EvictionLoopInterval=30s \
56+
--set shredder.ParkedNodeTTL=2m \
57+
--set shredder.RollingRestartThreshold=0.5 \
58+
--set shredder.EnableKarpenterDriftDetection=false \
59+
--set shredder.EnableNodeLabelDetection=true \
60+
--set shredder.NodeLabelsToDetect[0]="test-label" \
61+
--set shredder.NodeLabelsToDetect[1]="maintenance=scheduled" \
62+
--set shredder.NodeLabelsToDetect[2]="node.test.io/park" \
63+
--set logLevel=info \
64+
--set logFormat=text \
65+
--set dryRun=false \
66+
--set service.create=true \
67+
--set service.type=ClusterIP \
68+
--set service.port=8080 \
69+
--set service.targetPort=metrics
5370

5471
echo "KIND: deploying prometheus..."
5572
kubectl apply -f "${test_dir}/prometheus_stuffs_node_labels.yaml"
@@ -75,7 +92,7 @@ while [[ ${status} == *"False"* || -z ${status} ]]; do
7592
if [[ ${retry_count} == 600 ]]; then echo "Timeout exceeded!" && exit 1; fi
7693
# shellcheck disable=SC2059
7794
printf "\b${sp:i++%${#sp}:1}" && sleep 0.5;
78-
status=$(kubectl get pods -n kube-system -l app=k8s-shredder -o json | \
95+
status=$(kubectl get pods -n kube-system -l app.kubernetes.io/name=k8s-shredder -o json | \
7996
jq '.items[].status.conditions[] | select(.type=="Ready")| .status' 2> /dev/null)
8097
retry_count=$((retry_count+1))
8198
done
@@ -93,7 +110,7 @@ while [[ $(kubectl get pdb -n ns-team-k8s-shredder-test test-app-argo-rollout \
93110
done
94111

95112
echo ""
96-
kubectl logs -l app=k8s-shredder -n kube-system
113+
kubectl logs -l app.kubernetes.io/name=k8s-shredder -n kube-system
97114

98115
echo "K8S_SHREDDER: waiting for prometheus deployment to become ready!"
99116
retry_count=0
@@ -113,7 +130,7 @@ kubectl port-forward -n kube-system svc/k8s-shredder --kubeconfig=${KUBECONFIG_F
113130
It can take few minutes before seeing k8s-shredder metrics..."
114131

115132
echo -e "K8S_SHREDDER: You can access k8s-shredder logs by running
116-
kubectl logs -n kube-system -l app=k8s-shredder --kubeconfig=${KUBECONFIG_FILE} \n"
133+
kubectl logs -n kube-system -l app.kubernetes.io/name=k8s-shredder --kubeconfig=${KUBECONFIG_FILE} \n"
117134

118135
echo -e "K8S_SHREDDER: You can access prometheus metrics at http://localhost:1234 after running
119136
kubectl port-forward -n kube-system svc/prometheus --kubeconfig=${KUBECONFIG_FILE} 1234:9090\n"
@@ -140,4 +157,4 @@ kubectl label node "${WORKER_NODE2}" maintenance=scheduled
140157

141158
echo "NODE_LABELS: Labels applied! k8s-shredder should detect and park these nodes shortly..."
142159
echo "NODE_LABELS: You can monitor the process with:"
143-
echo " kubectl logs -n kube-system -l app=k8s-shredder --kubeconfig=${KUBECONFIG_FILE} -f"
160+
echo " kubectl logs -n kube-system -l app.kubernetes.io/name=k8s-shredder --kubeconfig=${KUBECONFIG_FILE} -f"

0 commit comments

Comments
 (0)