diff --git a/charts/dependencies/README.md b/charts/dependencies/README.md index 17198841..d5cb1857 100644 --- a/charts/dependencies/README.md +++ b/charts/dependencies/README.md @@ -277,7 +277,10 @@ elasticsearch: use_default_credentials: false ``` -For backward compatibility `HTTP_POST2_ALERT_URL` environment variable needs to be added to elastalert configuration. All alerts will be send to country config service and forwarded to email address +### Elastalert + + +For backward compatibility `HTTP_POST2_ALERT_URL` environment variable needs to be added to elastalert configuration. All alerts will be send to country config service and forwarded to email address defined while SMTP server configuration. See example: ```yaml @@ -288,7 +291,45 @@ elastalert: > NOTE: This behavior will be changed in future releases, see [#10608](https://github.com/opencrvs/opencrvs-core/issues/10608) - +**Custom rules** + +Elastalert rules can be extended by modifying or defining new rules. Rules can be stored as Kubernetes configmap within the same namespace as elastalert deployment. + +1. Create new folder and place rules there, e/g: + ``` + ~$ ls -1 rules/ + alert.yaml + log-alert-foo.yaml + log-error-bar.yaml + custom-service-error-foo.yaml + custom-service-error-bar.yaml + ssh-alert.yaml + ``` +2. Run following command to create configmap from rules: + ``` + kubectl create configmap elastalert-custom-rules \ + --from-file=charts/dependencies/files/elastalert/rules/ + ``` +3. Add `elastalert.custom_rules_configmap` to values.yaml to point elastalert to new configmap: + ```yaml + elastalert: + custom_rules_configmap: elastalert-custom-rules + ``` + +### Kibana + +Kibana has support for custom configuration shipped by default as config.ndjson file in helm chart: [charts/dependencies/files/kibana/config.ndjson](https://github.com/opencrvs/infrastructure/blob/develop/charts/dependencies/files/kibana/config.ndjson) + +If you need to customize that file please do following steps: +1. Create configmap from `config.ndjson` + ```bash + kubectl create cm kibana-custom-config --from-file config.ndjson + ``` +2. Add `kibana.custom_config_configmap` to values.yaml to point kibana to new configmap: + ```yaml + kibana: + custom_config_configmap: kibana-custom-config + ``` ## Backup Configuration The dependencies chart includes a built-in backup feature that supports automated backups for internal components. Backups are stored on an external server via an SSH connection. diff --git a/charts/dependencies/TODO.md b/charts/dependencies/TODO.md index 8572dc02..33035510 100644 --- a/charts/dependencies/TODO.md +++ b/charts/dependencies/TODO.md @@ -1,3 +1,2 @@ Open Questions: - Should we build dedicated helm chart for Monitoring? -- Move MinIO restore to cronjob diff --git a/charts/dependencies/files/kibana/setup.sh b/charts/dependencies/files/kibana/setup.sh index 0563e3ea..9474b416 100755 --- a/charts/dependencies/files/kibana/setup.sh +++ b/charts/dependencies/files/kibana/setup.sh @@ -80,7 +80,7 @@ _curl --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "$kibana done # Import configuration -_curl --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -X POST "${KIBANA_URL}/api/saved_objects/_import?overwrite=true" -H 'kbn-xsrf: true' --form file=@/scripts/config.ndjson > /dev/null +_curl --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD -X POST "${KIBANA_URL}/api/saved_objects/_import?overwrite=true" -H 'kbn-xsrf: true' --form file=@/config/config.ndjson > /dev/null # Re-enable all alerts _curl --connect-timeout 60 -u elastic:$ELASTICSEARCH_SUPERUSER_PASSWORD "$kibana_alerting_api_url" | jq -r '.data[].id' | while read -r id; do diff --git a/charts/dependencies/templates/elastalert.yaml b/charts/dependencies/templates/elastalert.yaml index 815937d7..dc7ac84e 100644 --- a/charts/dependencies/templates/elastalert.yaml +++ b/charts/dependencies/templates/elastalert.yaml @@ -62,11 +62,20 @@ spec: echo "processing file $f" envsubst < "$f" > "/rules/$(basename $f)" done + [ ! -d /rules-custom ] && exit 0 + for f in /rules-custom/*.yaml; do + echo "processing file $f" + envsubst < "$f" > "/rules/$(basename $f)" + done volumeMounts: - name: rules mountPath: /rules - name: rules-templates mountPath: /rules-templates + {{- if .Values.elastalert.custom_rules_configmap }} + - name: custom-rules + mountPath: /rules-custom + {{- end }} containers: - name: elastalert image: jertel/elastalert2:2.25.0 @@ -97,4 +106,9 @@ spec: - name: rules-templates configMap: name: elastalert-rules + {{- if .Values.elastalert.custom_rules_configmap }} + - name: rules-custom + configMap: + name: {{ .Values.elastalert.custom_rules_configmap }} + {{- end }} {{- end }} \ No newline at end of file diff --git a/charts/dependencies/templates/kibana-on-deploy-configmap.yaml b/charts/dependencies/templates/kibana-on-deploy-configmap.yaml new file mode 100644 index 00000000..a53dd826 --- /dev/null +++ b/charts/dependencies/templates/kibana-on-deploy-configmap.yaml @@ -0,0 +1,23 @@ +{{- if .Values.monitoring.enabled }} +apiVersion: v1 +data: + setup.sh: | +{{ .Files.Get "files/kibana/setup.sh" | indent 4 }} +kind: ConfigMap +metadata: + labels: + app: kibana + name: kibana-on-deploy-script +{{- if not .Values.kibana.custom_config_configmap }} +--- +apiVersion: v1 +data: + config.ndjson: | +{{ .Files.Get "files/kibana/config.ndjson" | indent 4 }} +kind: ConfigMap +metadata: + labels: + app: kibana + name: kibana-on-deploy-config +{{- end }} +{{- end }} diff --git a/charts/dependencies/templates/kibana-on-update.yaml b/charts/dependencies/templates/kibana-on-deploy.yaml similarity index 74% rename from charts/dependencies/templates/kibana-on-update.yaml rename to charts/dependencies/templates/kibana-on-deploy.yaml index b8c92e79..a8b26c1c 100644 --- a/charts/dependencies/templates/kibana-on-update.yaml +++ b/charts/dependencies/templates/kibana-on-deploy.yaml @@ -30,11 +30,17 @@ spec: key: ELASTIC_PASSWORD volumeMounts: - mountPath: /scripts - name: kibana-on-update-script + name: kibana-on-deploy-script + - mountPath: /config + name: kibana-configmap volumes: - - name: kibana-on-update-script + - name: kibana-on-deploy-script configMap: - name: kibana-on-update-script + name: kibana-on-deploy-script + defaultMode: 0755 + - name: kibana-configmap + configMap: + name: {{ .Values.kibana.custom_config_configmap | default "kibana-on-deploy-script" }} defaultMode: 0755 restartPolicy: "OnFailure" {{- end }} diff --git a/charts/dependencies/templates/kibana-on-update-configmap.yaml b/charts/dependencies/templates/kibana-on-update-configmap.yaml deleted file mode 100644 index e245879a..00000000 --- a/charts/dependencies/templates/kibana-on-update-configmap.yaml +++ /dev/null @@ -1,13 +0,0 @@ -{{- if .Values.monitoring.enabled }} -apiVersion: v1 -data: -{{- range $path, $file := .Files.Glob "files/kibana/*" }} - {{ base $path }}: | -{{ toString $file | indent 8 }} -{{- end }} -kind: ConfigMap -metadata: - labels: - app: kibana - name: kibana-on-update-script -{{- end }} diff --git a/charts/dependencies/values.yaml b/charts/dependencies/values.yaml index 5a2a8d56..3ac2eee0 100644 --- a/charts/dependencies/values.yaml +++ b/charts/dependencies/values.yaml @@ -76,6 +76,8 @@ monitoring: kibana: users_secret: kibana-users-secret + # Configure Kibana and persist updated configuration as configmap + # custom_config_configmap: metricbeat: dashboards: - kubernetes-e0195ce0-bcaf-11ec-b64f-7dd6e8e82013 @@ -83,6 +85,8 @@ filebeat: {} logstash: {} elastalert: + # Create configmap with custom rules for elastalert + # custom_rules_configmap: env: {} minio: