Add termination grace period and liveness probes (#65)

Signed-off-by: Kemal Akkoyun <[email protected]>

Author: kakkoyun

Diff for: examples/all/manifests/thanos-bucket-deployment.yaml

@@ -27,14 +27,29 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 8080
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-bucket
         ports:
         - containerPort: 8080
           name: http
+        readinessProbe:
+          httpGet:
+            path: /-/ready
+            port: 8080
+            scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: 250m
             memory: 512Mi
           requests:
             cpu: 100m
             memory: 256Mi
+      terminationGracePeriodSeconds: 120

Diff for: examples/all/manifests/thanos-compactor-statefulSet.yaml

@@ -32,6 +32,13 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-compactor
         ports:
         - containerPort: 10902
@@ -41,6 +48,8 @@ spec:
             path: /-/ready
             port: 10902
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: 500m
@@ -52,6 +61,7 @@ spec:
         - mountPath: /var/thanos/compactor
           name: thanos-compactor-data
           readOnly: false
+      terminationGracePeriodSeconds: 120
       volumes:
       - emptyDir: {}
         name: thanos-compactor-data

Diff for: examples/all/manifests/thanos-querier-deployment.yaml

@@ -26,6 +26,13 @@ spec:
         - --store=dnssrv+_grpc._tcp.prometheus-k8s.monitoring.svc.cluster.local
         - --store=dnssrv+_grpc._tcp.thanos-rule.monitoring.svc.cluster.local
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 9090
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-querier
         ports:
         - containerPort: 10901
@@ -37,10 +44,13 @@ spec:
             path: /-/ready
             port: 9090
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "1"
             memory: 1Gi
           requests:
             cpu: 100m
             memory: 256Mi
+      terminationGracePeriodSeconds: 120

Diff for: examples/all/manifests/thanos-receive-statefulSet.yaml

@@ -37,6 +37,13 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-receive
         ports:
         - containerPort: 10901
@@ -50,6 +57,8 @@ spec:
             path: /-/ready
             port: 10902
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "1"
@@ -61,6 +70,7 @@ spec:
         - mountPath: /var/thanos/receive
           name: thanos-receive-data
           readOnly: false
+      terminationGracePeriodSeconds: 120
   volumeClaimTemplates:
   - metadata:
       name: thanos-receive-data

Diff for: examples/all/manifests/thanos-store-statefulSet.yaml

@@ -30,6 +30,13 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-store
         ports:
         - containerPort: 10901
@@ -41,6 +48,8 @@ spec:
             path: /-/ready
             port: 10902
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "2"
@@ -52,6 +61,7 @@ spec:
         - mountPath: /var/thanos/store
           name: thanos-store-data
           readOnly: false
+      terminationGracePeriodSeconds: 120
   volumeClaimTemplates:
   - metadata:
       name: thanos-store-data

Diff for: jsonnet/kube-thanos/kube-thanos-bucket.libsonnet

@@ -50,12 +50,25 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.mixin.resources.withLimits({ cpu: '250m', memory: '512Mi' }) +
           container.withPorts([
             { name: 'http', containerPort: tb.ports.http },
-          ]);
+          ]) +
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.bucket.service.spec.ports[0].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.bucket.service.spec.ports[0].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         deployment.new(tb.name, 1, c, $.thanos.bucket.deployment.metadata.labels) +
         deployment.mixin.metadata.withNamespace(tb.namespace) +
         deployment.mixin.metadata.withLabels({ 'app.kubernetes.io/name': tb.name }) +
-        deployment.mixin.spec.selector.withMatchLabels(tb.labels),
+        deployment.mixin.spec.selector.withMatchLabels(tb.labels) +
+        deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120),
     },
   },
 }

Diff for: jsonnet/kube-thanos/kube-thanos-compactor.libsonnet

@@ -56,12 +56,24 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.withVolumeMounts([
             containerVolumeMount.new('thanos-compactor-data', '/var/thanos/compactor', false),
           ]) +
-          container.mixin.readinessProbe.httpGet.withPort($.thanos.compactor.service.spec.ports[0].port).withScheme('HTTP').withPath('/-/ready');
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.compactor.service.spec.ports[0].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.compactor.service.spec.ports[0].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         statefulSet.new(tc.name, 1, c, [], $.thanos.compactor.statefulSet.metadata.labels) +
         statefulSet.mixin.metadata.withNamespace(tc.namespace) +
         statefulSet.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.compactor.statefulSet.metadata.name }) +
         statefulSet.mixin.spec.withServiceName($.thanos.compactor.service.metadata.name) +
+        statefulSet.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) +
         statefulSet.mixin.spec.template.spec.withVolumes([
           volume.fromEmptyDir('thanos-compactor-data'),
         ]) +

Diff for: jsonnet/kube-thanos/kube-thanos-querier.libsonnet

@@ -46,12 +46,24 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
             { name: 'grpc', containerPort: $.thanos.querier.service.spec.ports[0].port },
             { name: 'http', containerPort: $.thanos.querier.service.spec.ports[1].port },
           ]) +
-          container.mixin.readinessProbe.httpGet.withPort($.thanos.querier.service.spec.ports[1].port).withScheme('HTTP').withPath('/-/ready');
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.querier.service.spec.ports[1].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.querier.service.spec.ports[1].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         deployment.new(tq.name, tq.replicas, c, $.thanos.querier.deployment.metadata.labels) +
         deployment.mixin.metadata.withNamespace(tq.namespace) +
         deployment.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.querier.deployment.metadata.name }) +
-        deployment.mixin.spec.selector.withMatchLabels($.thanos.querier.deployment.metadata.labels),
+        deployment.mixin.spec.selector.withMatchLabels($.thanos.querier.deployment.metadata.labels) +
+        deployment.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120),
     },
   },
 }

Diff for: jsonnet/kube-thanos/kube-thanos-receive.libsonnet

@@ -64,13 +64,25 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.withVolumeMounts([
             containerVolumeMount.new(tr.name + '-data', '/var/thanos/receive', false),
           ]) +
-          container.mixin.readinessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port).withScheme('HTTP').withPath('/-/ready');
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.receive.service.spec.ports[1].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         sts.new(tr.name, tr.replicas, c, [], $.thanos.receive.statefulSet.metadata.labels) +
         sts.mixin.metadata.withNamespace(tr.namespace) +
         sts.mixin.metadata.withLabels({ 'app.kubernetes.io/name': $.thanos.receive.statefulSet.metadata.name }) +
         sts.mixin.spec.withServiceName($.thanos.receive.service.metadata.name) +
         sts.mixin.spec.selector.withMatchLabels($.thanos.receive.statefulSet.metadata.labels) +
+        sts.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) +
         sts.mixin.spec.template.spec.withVolumes([
           volume.fromEmptyDir('data'),
         ]) +

Diff for: jsonnet/kube-thanos/kube-thanos-store.libsonnet

@@ -64,13 +64,25 @@ local k = import 'ksonnet/ksonnet.beta.4/k.libsonnet';
           container.withVolumeMounts([
             containerVolumeMount.new(ts.name + '-data', '/var/thanos/store', false),
           ]) +
-          container.mixin.readinessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port).withScheme('HTTP').withPath('/-/ready');
+          container.mixin.livenessProbe +
+          container.mixin.livenessProbe.withPeriodSeconds(30) +
+          container.mixin.livenessProbe.withFailureThreshold(4) +
+          container.mixin.livenessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port) +
+          container.mixin.livenessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.livenessProbe.httpGet.withPath('/-/healthy') +
+          container.mixin.readinessProbe +
+          container.mixin.readinessProbe.withInitialDelaySeconds(10) +
+          container.mixin.readinessProbe.withPeriodSeconds(30) +
+          container.mixin.readinessProbe.httpGet.withPort($.thanos.store.service.spec.ports[1].port) +
+          container.mixin.readinessProbe.httpGet.withScheme('HTTP') +
+          container.mixin.readinessProbe.httpGet.withPath('/-/ready');
 
         sts.new(ts.name, ts.replicas, c, [], $.thanos.store.statefulSet.metadata.labels) +
         sts.mixin.metadata.withNamespace(ts.namespace) +
         sts.mixin.metadata.withLabels({ 'app.kubernetes.io/name': ts.name }) +
         sts.mixin.spec.withServiceName($.thanos.store.service.metadata.name) +
         sts.mixin.spec.selector.withMatchLabels($.thanos.store.statefulSet.metadata.labels) +
+        sts.mixin.spec.template.spec.withTerminationGracePeriodSeconds(120) +
         sts.mixin.spec.template.spec.withVolumes([
           volume.fromEmptyDir(ts.name + '-data'),
         ]) +

Diff for: manifests/thanos-querier-deployment.yaml

@@ -23,6 +23,13 @@ spec:
         - --http-address=0.0.0.0:9090
         - --store=dnssrv+_grpc._tcp.thanos-store.monitoring.svc.cluster.local
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 9090
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-querier
         ports:
         - containerPort: 10901
@@ -34,10 +41,13 @@ spec:
             path: /-/ready
             port: 9090
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "1"
             memory: 1Gi
           requests:
             cpu: 100m
             memory: 256Mi
+      terminationGracePeriodSeconds: 120

Diff for: manifests/thanos-store-statefulSet.yaml

@@ -30,6 +30,13 @@ spec:
               key: thanos.yaml
               name: thanos-objectstorage
         image: quay.io/thanos/thanos:v0.9.0
+        livenessProbe:
+          failureThreshold: 4
+          httpGet:
+            path: /-/healthy
+            port: 10902
+            scheme: HTTP
+          periodSeconds: 30
         name: thanos-store
         ports:
         - containerPort: 10901
@@ -41,6 +48,8 @@ spec:
             path: /-/ready
             port: 10902
             scheme: HTTP
+          initialDelaySeconds: 10
+          periodSeconds: 30
         resources:
           limits:
             cpu: "2"
@@ -52,6 +61,7 @@ spec:
         - mountPath: /var/thanos/store
           name: thanos-store-data
           readOnly: false
+      terminationGracePeriodSeconds: 120
       volumes:
       - emptyDir: {}
         name: thanos-store-data