Merge pull request #179 from xrstf/etcd-probes

add probes to etcd statefulsets

Author: kcp-ci-bot

charts/kcp/Chart.yaml

@@ -3,7 +3,7 @@ name: kcp
 description: A prototype of a multi-tenant Kubernetes control plane for workloads on many clusters
 
 # version information
-version: 0.12.7
+version: 0.12.8
 appVersion: "0.28.3"
 
 # optional metadata

charts/kcp/templates/etcd-statefulset.yaml

@@ -1,4 +1,6 @@
-{{- if .Values.etcd.enabled -}}
+# This file is based on https://etcd.io/docs/v3.5/op-guide/kubernetes/
+
+{{ if .Values.etcd.enabled -}}
 apiVersion: v1
 kind: Service
 metadata:
@@ -15,6 +17,10 @@ spec:
     - port: 2380
       name: peer
       appProtocol: https
+    - port: 8080
+      name: metrics
+      appProtocol: http
+  publishNotReadyAddresses: true
   selector:
     {{- include "common.labels.selector" . | nindent 4 }}
     app.kubernetes.io/component: "etcd"
@@ -34,6 +40,9 @@ spec:
       {{- include "common.labels.selector" . | nindent 6 }}
       app.kubernetes.io/component: "etcd"
   replicas: 3
+  podManagementPolicy: Parallel
+  updateStrategy:
+    type: RollingUpdate
   template:
     metadata:
       labels:
@@ -68,6 +77,7 @@ spec:
           - --initial-cluster-token=etcd-cluster-1
           - --initial-cluster=$(PEERS)
           - --initial-cluster-state=new
+          - --listen-metrics-urls=http://0.0.0.0:8080
           - --auto-compaction-mode=periodic
           - --auto-compaction-retention=5m
           - --data-dir=/var/run/etcd/default.etcd
@@ -95,10 +105,29 @@ spec:
               name: client
             - containerPort: 2380
               name: peer
+            - containerPort: 8080
+              name: metrics
           {{- with .Values.etcd.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 30
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: 8080
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
           volumeMounts:
             - name: etcd-data
               mountPath: /var/run/etcd

charts/shard/Chart.yaml

@@ -3,7 +3,7 @@ name: shard
 description: A prototype of a multi-tenant Kubernetes control plane for workloads on many clusters
 
 # version information
-version: 0.0.5
+version: 0.0.6
 appVersion: "0.22.0"
 
 # optional metadata

charts/shard/templates/etcd-statefulset.yaml

@@ -1,4 +1,6 @@
-{{- if .Values.etcd.enabled -}}
+# This file is based on https://etcd.io/docs/v3.5/op-guide/kubernetes/
+
+{{ if .Values.etcd.enabled -}}
 apiVersion: v1
 kind: Service
 metadata:
@@ -11,8 +13,14 @@ spec:
   ports:
     - port: 2379
       name: client
+      appProtocol: https
     - port: 2380
       name: peer
+      appProtocol: https
+    - port: 8080
+      name: metrics
+      appProtocol: http
+  publishNotReadyAddresses: true
   selector:
     {{- include "common.labels.selector" . | nindent 4 }}
     app.kubernetes.io/component: "etcd"
@@ -32,12 +40,30 @@ spec:
       {{- include "common.labels.selector" . | nindent 6 }}
       app.kubernetes.io/component: "etcd"
   replicas: 3
+  podManagementPolicy: Parallel
+  updateStrategy:
+    type: RollingUpdate
   template:
     metadata:
       labels:
         {{- include "common.labels" . | nindent 8 }}
         app.kubernetes.io/component: "etcd"
     spec:
+      {{- with .Values.etcd.affinity}}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.etcd.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- if .Values.global.imagePullSecrets }}
+      imagePullSecrets:
+        {{- include "kcp.imagePullSecrets" . | trim | nindent 8 }}
+      {{- end }}
+      {{- with .Values.etcd.priorityClassName }}
+      priorityClassName: {{ . }}
+      {{- end }}
       containers:
         - name: etcd
           image: {{ .Values.etcd.image }}:{{ .Values.etcd.tag }}
@@ -51,6 +77,7 @@ spec:
           - --initial-cluster-token=etcd-cluster-1
           - --initial-cluster=$(PEERS)
           - --initial-cluster-state=new
+          - --listen-metrics-urls=http://0.0.0.0:8080
           - --auto-compaction-mode=periodic
           - --auto-compaction-retention=5m
           - --data-dir=/var/run/etcd/default.etcd
@@ -78,10 +105,29 @@ spec:
               name: client
             - containerPort: 2380
               name: peer
+            - containerPort: 8080
+              name: metrics
           {{- with .Values.etcd.resources }}
           resources:
             {{- toYaml . | nindent 12 }}
           {{- end }}
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: 8080
+            initialDelaySeconds: 10
+            periodSeconds: 5
+            timeoutSeconds: 5
+            successThreshold: 1
+            failureThreshold: 30
+          livenessProbe:
+            httpGet:
+              path: /livez
+              port: 8080
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
           volumeMounts:
             - name: etcd-data
               mountPath: /var/run/etcd
@@ -114,4 +160,7 @@ spec:
         resources:
           requests:
             storage: {{ .Values.etcd.volumeSize }}
+        {{- with .Values.etcd.storageClassName }}
+        storageClassName: {{ . }}
+        {{- end }}
 {{- end }}

charts/shard/values.yaml

@@ -19,8 +19,27 @@ etcd:
       # cpu: 1
      memory: 20Gi
   volumeSize: 8Gi
+  # StorageClass to use for etcd data volumes.
+  storageClassName: ""
   profiling:
     enabled: false
+  podDisruptionBudget:
+    enabled: false
+    maxUnavailable: 1
+  tolerations: []
+  priorityClassName: ""
+  affinity:
+    podAntiAffinity:
+      preferredDuringSchedulingIgnoredDuringExecution:
+        - weight: 1
+          podAffinityTerm:
+            labelSelector:
+              matchExpressions:
+                - key: "app.kubernetes.io/component"
+                  operator: In
+                  values:
+                    - etcd
+            topologyKey: "kubernetes.io/hostname"
 kcp:
   enabled: true
   sharding: