[loki] Generate Dashboards from loki-mixin

Signed-off-by: Jan-Otto Kröpke <mail@jkroepke.de>

Author: jkroepke

charts/loki/.gitignore

@@ -1 +1,12 @@
-charts/
+# Python development for hack
+venv
+pyvenv.cfg
+!**
+charts/*
+!charts/crds/
+!charts/crds/**
+Chart.lock
+hack/*.git
+hack/tmp/
+hack/venv/
+hack/pyvenv.cfg

charts/loki/.helmignore

@@ -30,3 +30,4 @@ test/
 scenarios/
 docs/
 *-values.yaml
+hack/

charts/loki/Chart.yaml

@@ -4,7 +4,7 @@ description: Helm chart for Grafana Loki supporting monolithic, simple scalable,
 type: application
 # renovate: docker=docker.io/grafana/loki
 appVersion: 3.7.1
-version: 9.3.3
+version: 9.5.0
 kubeVersion: ">=1.25.0-0"
 home: https://grafana-community.github.io/helm-charts
 sources:

charts/loki/ci/lint.sh

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+{
+  SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+
+  cd "${SCRIPT_DIR}/../hack/"
+
+  export PIP_DISABLE_PIP_VERSION_CHECK=1
+
+  python3 -m venv venv
+  # shellcheck disable=SC1091
+  source venv/bin/activate
+
+  pip3 install -r requirements.txt
+
+  go install -a github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest
+  PATH="$(go env GOPATH)/bin:$PATH"
+  export PATH
+
+  ./sync_grafana_dashboards.py
+  if ! git diff "$GITHUB_SHA" --color=always --exit-code; then
+    echo "Changes inside dashboards are not supported!"
+    echo "Please go into the ./hack/ directory and run ./sync_grafana_dashboards.py"
+    exit 1
+  fi
+
+  rm -rf ./venv ./*.git
+} 2>&1

charts/loki/ci/non-default-values.yaml

@@ -3,10 +3,13 @@ nameOverride: '{{ $.Release.Name }}'
 test:
   enabled: true
 loki:
+  auth_enabled: true
   commonConfig:
     replication_factor: 1
   useTestSchema: true
   tenants:
+    - name: "self-monitoring"
+      password: "self-monitoring"
     - name: "test-user-1"
       password: "test-password-1"
     - name: "test-user-2"
@@ -16,9 +19,42 @@ loki:
       - name: ndots
         value: "3"
 ingester:
-  replicas: 0
+  replicas: 1
   zoneAwareReplication:
     enabled: true
+    zoneA:
+      extraAffinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: name
+                    operator: In
+                    values:
+                      - '{{ include "loki.prefixIngesterName" . }}ingester-zone-a'
+              topologyKey: kubernetes.io/hostname
+    zoneB:
+      extraAffinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: name
+                    operator: In
+                    values:
+                      - '{{ include "loki.prefixIngesterName" . }}ingester-zone-b'
+              topologyKey: kubernetes.io/hostname
+    zoneC:
+      extraAffinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            - labelSelector:
+                matchExpressions:
+                  - key: name
+                    operator: In
+                    values:
+                      - '{{ include "loki.prefixIngesterName" . }}ingester-zone-c'
+              topologyKey: kubernetes.io/hostname
   dnsConfig:
     options:
       - name: ndots
@@ -51,6 +87,8 @@ queryScheduler:
 lokiCanary:
   enabled: true
   kind: Deployment
+  tenant:
+    password: "self-monitoring"
 chunksCache:
   suffix: "l1"
   enabled: true
@@ -80,9 +118,17 @@ gateway:
   metrics:
     enabled: false
 monitoring:
+  serviceMonitor:
+    enabled: true
+    labels:
+      release: kube-prometheus-stack
+  dashboards:
+    enabled: true
   rules:
     enabled: true
     alerting: true
+    labels:
+      release: kube-prometheus-stack
     additionalGroups:
       - name: "additional-rules"
         rules:

charts/loki/hack/README.md

@@ -0,0 +1,26 @@
+# loki charts hacks
+
+## [update_mixins.sh](update_mixins.sh)
+
+This script is a useful wrapper to run `sync_grafana_dashboards.py`.
+
+It clones all dependency dashboards into a tmp folder.
+
+And it lets you know if you are missing commandline-tools necessary for the
+update to complete.
+
+Therefore, if you want to create a PR that updates the mixins, please
+run `./hack/update_mixins.sh` from the charts directory
+(`./charts/loki`).
+
+## [sync_grafana_dashboards.py](sync_grafana_dashboards.py)
+
+This script generates grafana dashboards from JSON files, splitting them to separate files based on group name.
+
+List of imported dashboards:
+
+- [grafana/loki](https://github.com/grafana/loki/tree/main/production/loki-mixin) dashboards.
+    - In order to modify these dashboards:
+        - prepare and merge PR into [loki-mixin](https://github.com/grafana/loki/tree/main/production/loki-mixin) main and/or release branch.
+        - run sync_grafana_dashboards.py inside your fork of this repository
+        - send PR with changes to this repository

charts/loki/hack/renovate-post-upgrade-hook.sh

@@ -0,0 +1,35 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+SCRIPT_DIR=$(cd -- "$(dirname -- "${0}")" &>/dev/null && pwd)
+
+if ! which jb &>/dev/null; then
+  if ! which go &>/dev/null; then
+    echo "'jb' command not found"
+    echo "Install jsonnet-bundler from https://github.com/jsonnet-bundler/jsonnet-bundler"
+    exit 1
+  fi
+
+  echo "'jb' command not found. Try to install it from github.com/jsonnet-bundler/jsonnet-bundler"
+
+  go install -a github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb@latest
+  PATH="$(go env GOPATH)/bin:$PATH"
+  export PATH
+
+  if ! which jb &>/dev/null; then
+    echo "'jb' command not found"
+    echo "Install jsonnet-bundler from https://github.com/jsonnet-bundler/jsonnet-bundler"
+    exit 1
+  fi
+fi
+
+export PIP_DISABLE_PIP_VERSION_CHECK=1
+
+python3 -m venv "${SCRIPT_DIR}/tmp/venv"
+# shellcheck disable=SC1091
+source "${SCRIPT_DIR}/tmp/venv/bin/activate"
+
+pip3 install -r "${SCRIPT_DIR}/requirements.txt"
+
+"${SCRIPT_DIR}/sync_grafana_dashboards.py"

charts/loki/hack/requirements.txt

@@ -0,0 +1,3 @@
+PyYAML==6.0.3
+requests==2.32.5
+jsonnet==0.21.0