Merge pull request #223 from scality/improvement/kibana-index

Pre-provision Kibana index configuration

Author: alxf

ChangeLog.rst

@@ -26,6 +26,8 @@ Kubespray
 
 :ghpull:`218` - update versions of Kibana and `fluent-bit`
 
+:ghpull:`223` - pre-provision Kibana index configuration (:ghissue:`174`)
+
 Bugs fixed
 ----------
 :ghissue:`220` - 'Kubernetes Calico (Alternative)' dashboard doesn't work (:ghpull:`221`)

roles/cluster_logging/defaults/main.yml

@@ -37,6 +37,7 @@ kibana_version: '0.10.0'
 kibana_repo: 'https://kubernetes-charts.storage.googleapis.com'
 kibana_namespace: 'kube-ops'
 kibana_release_name: 'kibana'
+kibana_addons_dir: '/etc/kubernetes/addons/kibana'
 
 elasticsearch_exporter_chart: 'elasticsearch-exporter'
 elasticsearch_exporter_version: '0.2.0'

roles/cluster_logging/files/kibana/provision-index.sh

@@ -0,0 +1,69 @@
+#!/bin/sh
+
+#
+# Script to setup the default index pattern in kibana
+#
+
+set -ue
+
+INDEX_ID="logstash-*"
+
+API_HOSTNAME="$KIBANA_SERVICE_HOST:$KIBANA_SERVICE_PORT"
+
+COUNTDOWN=10
+SLEEP_DELAY=5
+
+set_index_pattern()
+{
+    curl -s -o /dev/null -w "%{http_code}" \
+        -XPOST \
+        -H "Content-type: application/json" \
+        -H "kbn-xsrf: anything" \
+        "http://$API_HOSTNAME/api/saved_objects/index-pattern/$INDEX_ID" \
+        -d "{\"attributes\": {\"title\": \"$INDEX_ID\", \"timeFieldName\": \"@timestamp\"}}"
+}
+
+set_default_index()
+{
+    curl -s -o /dev/null -w "%{http_code}" \
+        -XPOST \
+        -H "Content-type: application/json" \
+        -H "kbn-xsrf: anything" \
+        "http://$API_HOSTNAME/api/kibana/settings/defaultIndex" \
+        -d "{\"value\": \"$INDEX_ID\"}"
+}
+
+
+# Wait for kibana service to be ready
+while [ "$COUNTDOWN" -gt 0 ]; do
+    http_code=$(curl -s -o /dev/null -w "%{http_code}" "$API_HOSTNAME")
+    if [ "$http_code" = "200" ]; then
+        break
+    else
+        COUNTDOWN=$((COUNTDOWN - 1))
+        if [ "$COUNTDOWN" -le 0 ]; then
+            echo "Error: Can't connect to the Kibana API server"
+            exit 1
+        else
+            sleep $SLEEP_DELAY
+        fi
+    fi
+done
+
+http_code=$(set_index_pattern)
+
+if [ "$http_code" = "409" ]; then
+    echo "Index pattern '$INDEX_ID' is already registered"
+elif [ "$http_code" != "200" ]; then
+    echo "Error: An error occurred when trying to set the Kibana index pattern (http code: $http_code)"
+    exit 1
+fi
+
+http_code=$(set_default_index)
+
+if [ "$http_code" != "200" ]; then
+    echo "Error: An error occurred when trying to set the default Kibana index (http code: $http_code)"
+    exit 1
+fi
+
+echo "Successful registered '$INDEX_ID' as Kibana index pattern"

roles/cluster_logging/files/kibana/values.yml

@@ -6,3 +6,4 @@ service:
 env:
   ELASTICSEARCH_URL: 'http://elasticsearch-client:9200'
   SERVER_BASEPATH: '/api/v1/namespaces/kube-ops/services/http:kibana:/proxy'
+  SERVER_DEFAULTROUTE: '/app/kibana#/discover'

roles/cluster_logging/tasks/kibana-index-provisioning.yml

@@ -0,0 +1,24 @@
+---
+
+- name: 'create remote addons directory'
+  file:
+    state: directory
+    path: '{{ kibana_addons_dir }}'
+
+- name: 'push addons files'
+  template:
+    src: '{{ item }}.j2'
+    dest: '{{ kibana_addons_dir }}/{{ item }}'
+  register: kibana_setup_manifests
+  with_items:
+    - kibana-index-provisioning-configmap.yml
+    - kibana-index-provisioning-job.yml
+
+- name: 'apply the Kibana job setup'
+  run_once: true
+  kube:
+    kubectl: '{{ bin_dir }}/kubectl'
+    filename: '{{ item.dest }}'
+    namespace: '{{ kibana_namespace }}'
+    state: 'latest'
+  with_items: '{{ kibana_setup_manifests.results | default([]) }}'

roles/cluster_logging/tasks/kibana.yml

@@ -32,3 +32,6 @@
     dest: '{{ kibana_values_file.path }}'
     state: absent
   when: remove_metal_k8s_temporary_file|bool
+
+- name: 'include Kibana index provisioning job'
+  import_tasks: kibana-index-provisioning.yml

roles/cluster_logging/templates/kibana-index-provisioning-configmap.yml.j2

@@ -0,0 +1,11 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: kibana-index-provisioning
+  labels:
+    app: kibana
+    heritage: MetalK8s
+    component: index-provisioning
+data:
+  provision-index.sh: |-
+    {{ lookup('file', role_path ~ '/files/kibana/provision-index.sh') | indent(8) }}

roles/cluster_logging/templates/kibana-index-provisioning-job.yml.j2

@@ -0,0 +1,30 @@
+apiVersion: batch/v1
+kind: Job
+metadata:
+  name: kibana-index-provisioning
+  labels:
+    app: kibana
+    heritage: MetalK8s
+    component: index-provisioning
+spec:
+  template:
+    metadata:
+      name: kibana-index-provisioning
+      labels:
+        app: kibana
+        heritage: MetalK8s
+        component: index-provisioning
+    spec:
+      containers:
+        - name: provision-index
+          image: appropriate/curl
+          command: ['sh', '/scripts/provision-index.sh']
+          volumeMounts:
+            - name: scripts
+              mountPath: /scripts
+              readOnly: true
+      restartPolicy: OnFailure
+      volumes:
+        - name: scripts
+          configMap:
+            name: kibana-index-provisioning