Cassandra management

tools/cassandra/backup-medusa/README.md

@@ -0,0 +1,9 @@
+# Medusa for Apache Cassandra™
+
+Medusa is an Apache Cassandra backup system.
+
+https://github.com/thelastpickle/cassandra-medusa
+
+This repo is an example from the real workout on Kubernetes
+Here are provided an examples and initial patch for apache libcloud to support S3-compatible storages with SSL.
+For now, it might be already in standard Medusa as our patch is already merged to the libcloud

tools/cassandra/backup-medusa/cassandra-medusa-daily-backup-cronjob.yml

@@ -0,0 +1,178 @@
+# test this job:
+# kubectl create job --from=cronjob/cassandra-daily-backup-0 cassandra-daily-backup-0-manual
+# for i in {0..2}; do kubectl create job --from=cronjob/cassandra-daily-backup-${i} cassandra-daily-backup-${i}-manual; sleep 30; done
+# kubectl delete job cassandra-daily-backup-0-manual
+# for i in {0..2}; do kubectl delete job cassandra-daily-backup-${i}-manual; done
+# curl --noproxy '*' -X DELETE http://prometheus-pushgateway.${system}.svc.cluster.local:9091/metrics/job/cronjob/instance/${CRON_JOB}
+
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: cassandra-backup
+data:
+  cassandra-backup.sh: |+
+    #!/bin/bash
+    set -e
+    echo "[POD] Starting backup with kubectl exec..."
+    POD_ID=$(echo $POD_NAME | grep -o "[0-9]*" | head -1)
+    BACKUP_NAME=$(date +%F)
+    echo "[POD] backup name is [$BACKUP_NAME] for pod [$POD_ID]"
+    kubectl exec -c cassandra cassandra-helm-$POD_ID -- medusa -v backup --backup-name=$BACKUP_NAME --mode=differential
+    CRON_JOB=$(echo $POD_NAME | grep -o "[a-z][a-z-]*[a-z]" | head -1)
+    CRON_JOB=${CRON_JOB}-${POD_ID}
+    echo "Pushing metrics for cron job $CRON_JOB..."
+    cat << EOF | curl --data-binary @- http://prometheus-pushgateway.${NAMESPACE}.svc.cluster.local:9091/metrics/job/cronjob/instance/${CRON_JOB}
+    # HELP job_executed_successful A job execution status
+    # TYPE job_executed_successful gauge
+    job_executed_successful 1
+    EOF
+    echo "[POD] Done."
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cassandra-daily-backup-0
+spec:
+  schedule: "0 0 * * *"
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    backoffLimit: 1 # two runs
+    spec:
+      template:
+        spec:
+          serviceAccountName: tb-exec-service-account
+          volumes:
+            - name: cassandra-backup
+              configMap:
+                name: cassandra-backup
+                defaultMode: 0755
+          containers:
+            - name: backup
+              image: bitnami/kubectl:1.23.6-debian-10-r25
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: NAMESPACE
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.namespace
+                - name: POD_NAME
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.name
+              volumeMounts:
+                - name: cassandra-backup
+                  mountPath: /tmp/cassandra-backup.sh
+                  subPath: cassandra-backup.sh
+              command:
+                - /bin/bash
+                - -ec
+                - /tmp/cassandra-backup.sh
+              resources:
+                limits:
+                  cpu: 100m
+                  memory: 100Mi
+                requests:
+                  cpu: 100m
+                  memory: 100Mi
+          restartPolicy: OnFailure
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cassandra-daily-backup-1
+spec:
+  schedule: "1 0 * * *"
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    backoffLimit: 1 # two runs
+    spec:
+      template:
+        spec:
+          serviceAccountName: tb-exec-service-account
+          volumes:
+            - name: cassandra-backup
+              configMap:
+                name: cassandra-backup
+                defaultMode: 0755
+          containers:
+            - name: backup
+              image: bitnami/kubectl:1.23.6-debian-10-r25
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: NAMESPACE
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.namespace
+                - name: POD_NAME
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.name
+              volumeMounts:
+                - name: cassandra-backup
+                  mountPath: /tmp/cassandra-backup.sh
+                  subPath: cassandra-backup.sh
+              command:
+                - /bin/bash
+                - -ec
+                - /tmp/cassandra-backup.sh
+              resources:
+                limits:
+                  cpu: 100m
+                  memory: 100Mi
+                requests:
+                  cpu: 100m
+                  memory: 100Mi
+          restartPolicy: OnFailure
+---
+apiVersion: batch/v1
+kind: CronJob
+metadata:
+  name: cassandra-daily-backup-2
+spec:
+  schedule: "2 0 * * *"
+  successfulJobsHistoryLimit: 3
+  failedJobsHistoryLimit: 2
+  jobTemplate:
+    backoffLimit: 1 # two runs
+    spec:
+      template:
+        spec:
+          serviceAccountName: tb-exec-service-account
+          volumes:
+            - name: cassandra-backup
+              configMap:
+                name: cassandra-backup
+                defaultMode: 0755
+          containers:
+            - name: backup
+              image: bitnami/kubectl:1.23.6-debian-10-r25
+              imagePullPolicy: IfNotPresent
+              env:
+                - name: NAMESPACE
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.namespace
+                - name: POD_NAME
+                  valueFrom:
+                    fieldRef:
+                      fieldPath: metadata.name
+              volumeMounts:
+                - name: cassandra-backup
+                  mountPath: /tmp/cassandra-backup.sh
+                  subPath: cassandra-backup.sh
+              command:
+                - /bin/bash
+                - -ec
+                - /tmp/cassandra-backup.sh
+              resources:
+                limits:
+                  cpu: 100m
+                  memory: 100Mi
+                requests:
+                  cpu: 100m
+                  memory: 100Mi
+          restartPolicy: OnFailure
+---
+## add more for more than 3-nodes cluster

tools/cassandra/backup-medusa/docker-custom/build_and_deploy.sh

@@ -0,0 +1,54 @@
+#
+# ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL
+#
+# Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved.
+#
+# NOTICE: All information contained herein is, and remains
+# the property of ThingsBoard, Inc. and its suppliers,
+# if any.  The intellectual and technical concepts contained
+# herein are proprietary to ThingsBoard, Inc.
+# and its suppliers and may be covered by U.S. and Foreign Patents,
+# patents in process, and are protected by trade secret or copyright law.
+#
+# Dissemination of this information or reproduction of this material is strictly forbidden
+# unless prior written permission is obtained from COMPANY.
+#
+# Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees,
+# managers or contractors who have executed Confidentiality and Non-disclosure agreements
+# explicitly covering such access.
+#
+# The copyright notice above does not evidence any actual or intended publication
+# or disclosure  of  this source code, which includes
+# information that is confidential and/or proprietary, and is a trade secret, of  COMPANY.
+# ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC  PERFORMANCE,
+# OR PUBLIC DISPLAY OF OR THROUGH USE  OF THIS  SOURCE CODE  WITHOUT
+# THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED,
+# AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES.
+# THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION
+# DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS,
+# OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT  MAY DESCRIBE, IN WHOLE OR IN PART.
+#
+
+set -e
+
+version=medusa-13-r09
+registry_src=docker.io/bitnami
+image_src=cassandra:4.0.3-debian-10-r68
+registry=docker.io/sevlamat
+image=$image_src-$version
+
+echo Building image $image ...
+
+docker pull $registry_src/$image_src
+
+# --no-cache
+docker build -t $image .
+
+docker tag $image $registry/$image
+
+read -n 1 -p "Press any key to push to $registry/$image or ^C to stop... "
+
+docker push $registry/$image
+
+echo Build completed.
+

tools/cassandra/backup-medusa/docker-custom/credentials

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id = Put0Your0Key0Id0Here
+aws_secret_access_key = Put0Your0Secret0Here

tools/cassandra/backup-medusa/docker-custom/credentials_template

@@ -0,0 +1,3 @@
+[default]
+aws_access_key_id = $AWS_ACCESS_KEY_ID
+aws_secret_access_key = $AWS_SECRET_ACCESS_KEY

tools/cassandra/backup-medusa/docker-custom/dockerfile

@@ -0,0 +1,68 @@
+#
+# ThingsBoard, Inc. ("COMPANY") CONFIDENTIAL
+#
+# Copyright © 2016-2021 ThingsBoard, Inc. All Rights Reserved.
+#
+# NOTICE: All information contained herein is, and remains
+# the property of ThingsBoard, Inc. and its suppliers,
+# if any.  The intellectual and technical concepts contained
+# herein are proprietary to ThingsBoard, Inc.
+# and its suppliers and may be covered by U.S. and Foreign Patents,
+# patents in process, and are protected by trade secret or copyright law.
+#
+# Dissemination of this information or reproduction of this material is strictly forbidden
+# unless prior written permission is obtained from COMPANY.
+#
+# Access to the source code contained herein is hereby forbidden to anyone except current COMPANY employees,
+# managers or contractors who have executed Confidentiality and Non-disclosure agreements
+# explicitly covering such access.
+#
+# The copyright notice above does not evidence any actual or intended publication
+# or disclosure  of  this source code, which includes
+# information that is confidential and/or proprietary, and is a trade secret, of  COMPANY.
+# ANY REPRODUCTION, MODIFICATION, DISTRIBUTION, PUBLIC  PERFORMANCE,
+# OR PUBLIC DISPLAY OF OR THROUGH USE  OF THIS  SOURCE CODE  WITHOUT
+# THE EXPRESS WRITTEN CONSENT OF COMPANY IS STRICTLY PROHIBITED,
+# AND IN VIOLATION OF APPLICABLE LAWS AND INTERNATIONAL TREATIES.
+# THE RECEIPT OR POSSESSION OF THIS SOURCE CODE AND/OR RELATED INFORMATION
+# DOES NOT CONVEY OR IMPLY ANY RIGHTS TO REPRODUCE, DISCLOSE OR DISTRIBUTE ITS CONTENTS,
+# OR TO MANUFACTURE, USE, OR SELL ANYTHING THAT IT  MAY DESCRIBE, IN WHOLE OR IN PART.
+#
+
+FROM docker.io/bitnami/cassandra:4.0.3-debian-10-r68
+
+USER root
+
+#RUN mkdir /.config/htop
+COPY --chown=1001:1001 htoprc /.config/htop/
+
+# additional software added. the most useful is htop that shows the disk IO, CPU and mem
+RUN apt-get update && apt-get install -y \
+    bash-completion ncdu mc \
+    htop curl wget net-tools iputils-ping traceroute \
+    python3-pip \
+    && rm -rf /var/lib/apt/lists/*
+
+RUN mkdir -p /.local/bin /.cache/pip /.config/mc
+
+RUN pip3 install cassandra-medusa[S3]
+
+#Apply patch for content-type issue (temporary, we already contributed to the meduse and this have to be in the latest already)
+#COPY s3.py /opt/bitnami/python/lib/python3.9/site-packages/libcloud/storage/drivers/s3.py
+#RUN chmod a+rw /opt/bitnami/python/lib/python3.9/site-packages/libcloud/storage/drivers/s3.py
+
+COPY --chown=1001:1001 medusa_template.ini /etc/medusa/
+COPY --chown=1001:1001 credentials_template /etc/medusa/
+# Put your truustcert here if necessary for custom S3 provider
+#COPY --chown=1001:1001 trustcerts.pem /etc/medusa/
+
+RUN chown -R 1001:1001 /.local /.cache /.config \
+    $(python3 -m certifi)
+RUN chmod -R a+rw /.local /.cache /etc/medusa /.config \
+    $(python3 -m certifi)
+
+#  && mkdir -p /.cache/pip && chown -R 1001:1001 /.cache
+ENV PATH="${PATH}:/.local/bin"
+#check user and cmd on respective container source
+# https://github.com/bitnami/bitnami-docker-cassandra/blob/3.11.11-debian-10-r63/3.11/debian-10/Dockerfile
+USER 1001

tools/cassandra/backup-medusa/docker-custom/htoprc

@@ -0,0 +1,26 @@
+# Beware! This file is rewritten by htop when settings are changed in the interface.
+# The parser is also very primitive, and not human-friendly.
+fields=0 48 17 18 38 39 40 109 110 111 2 46 47 49 1
+sort_key=111
+sort_direction=1
+hide_threads=0
+hide_kernel_threads=1
+hide_userland_threads=0
+shadow_other_users=0
+show_thread_names=1
+show_program_path=1
+highlight_base_name=0
+highlight_megabytes=1
+highlight_threads=1
+tree_view=0
+header_margin=1
+detailed_cpu_time=0
+cpu_count_from_zero=0
+update_process_names=0
+account_guest_in_cpu_meter=0
+color_scheme=0
+delay=15
+left_meters=LeftCPUs Memory Swap
+left_meter_modes=1 1 1
+right_meters=RightCPUs Tasks LoadAverage Uptime
+right_meter_modes=1 2 2 2