feat: Provide way to configure alerting rules

Author: adskyiproger

charts/dependencies/README.md

@@ -277,7 +277,10 @@ elasticsearch:
   use_default_credentials: false
 ```
 
-For backward compatibility `HTTP_POST2_ALERT_URL` environment variable needs to be added to elastalert configuration. All alerts will be send to country config service and forwarded to email address
+### Elastalert
+
+
+For backward compatibility `HTTP_POST2_ALERT_URL` environment variable needs to be added to elastalert configuration. All alerts will be send to country config service and forwarded to email address defined while SMTP server configuration.
 
 See example:
 ```yaml
@@ -288,7 +291,45 @@ elastalert:
 
 > NOTE: This behavior will be changed in future releases, see [#10608](https://github.com/opencrvs/opencrvs-core/issues/10608)
 
-
+**Custom rules**
+
+Elastalert rules can be extended by modifying or defining new rules. Rules can be stored as Kubernetes configmap within the same namespace as elastalert deployment. 
+
+1. Create new folder and place rules there, e/g:
+    ```
+    ~$ ls -1 rules/
+    alert.yaml
+    log-alert-foo.yaml
+    log-error-bar.yaml
+    custom-service-error-foo.yaml
+    custom-service-error-bar.yaml
+    ssh-alert.yaml
+    ```
+2. Run following command to create configmap from rules:
+    ```
+    kubectl create configmap elastalert-custom-rules \
+        --from-file=charts/dependencies/files/elastalert/rules/
+    ```
+3. Add `elastalert.custom_rules_configmap` to values.yaml to point elastalert to new configmap:
+    ```yaml
+    elastalert:
+      custom_rules_configmap: elastalert-custom-rules
+    ```
+
+### Kibana
+
+Kibana has support for custom configuration shipped by default as config.ndjson file in helm chart: [charts/dependencies/files/kibana/config.ndjson](https://github.com/opencrvs/infrastructure/blob/develop/charts/dependencies/files/kibana/config.ndjson)
+
+If you need to customize that file please do following steps:
+1. Create configmap from `config.ndjson`
+   ```bash
+   kubectl create cm kibana-custom-config --from-file config.ndjson 
+   ```
+2. Add `kibana.custom_config_configmap` to values.yaml to point  kibana to new configmap:
+   ```yaml
+   kibana:
+     custom_config_configmap: kibana-custom-config
+   ```
 ## Backup Configuration
 
 The dependencies chart includes a built-in backup feature that supports automated backups for internal components. Backups are stored on an external server via an SSH connection.

charts/dependencies/TODO.md

@@ -1,3 +1,2 @@
 Open Questions:
 - Should we build dedicated helm chart for Monitoring?
-- Move MinIO restore to cronjob

charts/dependencies/files/kibana/setup.sh

@@ -80,7 +80,7 @@ _curl --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "$kibana
 done
 
 # Import configuration
-_curl --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -X POST "${KIBANA_URL}/api/saved_objects/_import?overwrite=true" -H 'kbn-xsrf: true' --form file=@/scripts/config.ndjson > /dev/null
+_curl --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -X POST "${KIBANA_URL}/api/saved_objects/_import?overwrite=true" -H 'kbn-xsrf: true' --form file=@/config/config.ndjson > /dev/null
 
 # Re-enable all alerts
 _curl --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "$kibana_alerting_api_url" | jq -r '.data[].id' | while read -r id; do

charts/dependencies/templates/elastalert.yaml

@@ -62,11 +62,20 @@ spec:
                 echo "processing file $f"
                 envsubst < "$f" > "/rules/$(basename $f)"
               done
+              [ ! -d /rules-custom ] && exit 0
+              for f in /rules-custom/*.yaml; do
+                echo "processing file $f"
+                envsubst < "$f" > "/rules/$(basename $f)"
+              done              
           volumeMounts:
             - name: rules
               mountPath: /rules
             - name: rules-templates
               mountPath: /rules-templates
+          {{- if .Values.elastalert.custom_rules_configmap }}
+            - name: custom-rules
+              mountPath: /rules-custom
+          {{- end }}
       containers:
         - name: elastalert
           image: jertel/elastalert2:2.25.0
@@ -97,4 +106,9 @@ spec:
         - name: rules-templates
           configMap:
             name: elastalert-rules
+      {{- if .Values.elastalert.custom_rules_configmap }}
+        - name: rules-custom
+          configMap:
+            name: {{ .Values.elastalert.custom_rules_configmap }}
+      {{- end }}
 {{- end }}

charts/dependencies/templates/kibana-on-deploy-configmap.yaml

@@ -0,0 +1,23 @@
+{{- if .Values.monitoring.enabled }}
+apiVersion: v1
+data:
+  setup.sh: |
+{{ .Files.Get "files/kibana/setup.sh" | indent 4 }}
+kind: ConfigMap
+metadata:
+  labels:
+    app: kibana
+  name: kibana-on-deploy-script
+{{- if not .Values.kibana.custom_config_configmap }}
+---
+apiVersion: v1
+data:
+  config.ndjson: |
+{{ .Files.Get "files/kibana/config.ndjson" | indent 4 }}
+kind: ConfigMap
+metadata:
+  labels:
+    app: kibana
+  name: kibana-on-deploy-config
+{{- end }}
+{{- end }}

…ndencies/templates/kibana-on-update.yaml …ndencies/templates/kibana-on-deploy.yamlcharts/dependencies/templates/kibana-on-update.yaml renamed to charts/dependencies/templates/kibana-on-deploy.yaml charts/dependencies/templates/kibana-on-update.yaml renamed to charts/dependencies/templates/kibana-on-deploy.yaml

@@ -30,11 +30,17 @@ spec:
                   key: ELASTIC_PASSWORD
           volumeMounts:
             - mountPath: /scripts
-              name: kibana-on-update-script
+              name: kibana-on-deploy-script
+            - mountPath: /config
+              name: kibana-configmap
       volumes:
-        - name: kibana-on-update-script
+        - name: kibana-on-deploy-script
           configMap:
-            name: kibana-on-update-script
+            name: kibana-on-deploy-script
+            defaultMode: 0755
+        - name: kibana-configmap
+          configMap:
+            name: {{ .Values.kibana.custom_config_configmap | default "kibana-on-deploy-script" }}
             defaultMode: 0755
       restartPolicy: "OnFailure"
 {{- end }}

charts/dependencies/templates/kibana-on-update-configmap.yaml

@@ -76,13 +76,17 @@ monitoring:
 
 kibana:
   users_secret: kibana-users-secret
+  # Configure Kibana and persist updated configuration as configmap
+  # custom_config_configmap:
 metricbeat:
   dashboards:
     - kubernetes-e0195ce0-bcaf-11ec-b64f-7dd6e8e82013
 filebeat: {}
 logstash: {}
 
 elastalert:
+  # Create configmap with custom rules for elastalert
+  # custom_rules_configmap: 
   env: {}
 
 minio: