Deploy more logstash and apm-server replicas (#3784)

* deploy more logstash and apm-server replicas, use logstash output for all services including dependencies

* add restart: always for apm-server

* simplify 7-day-default ILM config so it always deletes records after 7 days or 300mb

* make apm-rollover-7-days policy delete its data after 7 days

* add default for number of replicates for indexes as 1

* move replica settings to init script as 7.x doesn't support it in .yml file

* add logAlias to kibana config

* only deploy 2 instances of apm-server and logstash

Author: rikukissa

docker-compose.deploy.yml

@@ -31,7 +31,11 @@ services:
           - node.role == manager
     networks:
       - overlay_net
-
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'traefik'
   metricbeat:
     image: docker.elastic.co/beats/metricbeat:7.17.0
 
@@ -62,7 +66,11 @@ services:
         target: /usr/share/metricbeat/metricbeat.yml
       - source: metricbeat-rollover-policy.{{ts}}
         target: /usr/share/metricbeat/rollover-policy.json
-
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'metricbeat'
   setup-kibana-config:
     image: curlimages/curl:7.83.1
     entrypoint:
@@ -90,7 +98,11 @@ services:
       placement:
         constraints:
           - node.role == manager
-
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'setup-kibana-config'
   kibana:
     image: docker.elastic.co/kibana/kibana:7.17.0
     restart: always
@@ -114,7 +126,11 @@ services:
         target: /usr/share/kibana/config/kibana.yml
     depends_on:
       - elasticsearch
-
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'kibana'
   # Configure mongo nodes as a replica set
   mongo1:
     command: mongod --auth --replSet rs0 --keyFile /etc/mongodb-keyfile
@@ -150,7 +166,11 @@ services:
         target: /docker-entrypoint-initdb.d/create-mongo-users.sh
     networks:
       - overlay_net
-
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'mongo1'
   mongo-on-update:
     image: mongo:4.4
     command: bash /on-deploy.sh
@@ -174,12 +194,20 @@ services:
       - WEBHOOKS_MONGODB_PASSWORD=${WEBHOOKS_MONGODB_PASSWORD}
     networks:
       - overlay_net
-
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'mongo-on-update'
   # Configure redis
   redis:
     networks:
       - overlay_net
-
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'redis'
   # Configure elasticsearch
   elasticsearch:
     volumes:
@@ -203,10 +231,15 @@ services:
       - 9200:9200
     networks:
       - overlay_net
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'elasticsearch'
 
   setup-elasticsearch-users:
     image: ubuntu:bionic
-    entrypoint: ['bash', '/usr/app/setup-users.sh']
+    entrypoint: ['bash', '/usr/app/setup.sh']
     restart: on-failure
     depends_on:
       - elasticsearch
@@ -228,6 +261,11 @@ services:
       placement:
         constraints:
           - node.role == manager
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'setup-elasticsearch-users'
   elastalert:
     image: jertel/elastalert2:2.3.0
     restart: unless-stopped
@@ -241,6 +279,11 @@ services:
       placement:
         constraints:
           - node.role == manager
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'elastalert'
 
   logstash:
     image: logstash:7.17.0
@@ -263,13 +306,16 @@ services:
     networks:
       - overlay_net
     restart: always
+    deploy:
+      replicas: 2
   apm-server:
     image: docker.elastic.co/apm/apm-server:7.15.2
     depends_on:
       - elasticsearch
       - kibana
     cap_add: ['CHOWN', 'DAC_OVERRIDE', 'SETGID', 'SETUID']
     cap_drop: ['ALL']
+    restart: always
     ports:
       - 8200:8200
     networks:
@@ -294,7 +340,11 @@ services:
     configs:
       - source: apm.{{ts}}
         target: /usr/share/apm-server/apm-server.yml
-
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'apm-server'
   # Configure influxdb
   influxdb:
     volumes:
@@ -307,7 +357,11 @@ services:
           - node.labels.data1 == true
     networks:
       - overlay_net
-
+    logging:
+      driver: gelf
+      options:
+        gelf-address: 'udp://127.0.0.1:12201'
+        tag: 'influxdb'
   # Configure webapps and add traefik config
   client:
     environment:

infrastructure/elasticsearch/setup-helpers.sh

@@ -198,5 +198,36 @@ function ensure_role {
     echo -e "\n${output::-3}\n"
   fi
 
+  return $result
+}
+
+function ensure_settings {
+  local body=$1
+
+  local elasticsearch_host="${ELASTICSEARCH_HOST:-elasticsearch}"
+
+  local -a args=( '-s' '-D-' '-m15' '-w' '%{http_code}'
+    "http://${elasticsearch_host}:9200/_settings"
+    '-X' 'PUT'
+    '-H' 'Content-Type: application/json'
+    '-d' "$body"
+    )
+
+  if [[ -n "${ELASTIC_PASSWORD:-}" ]]; then
+    args+=( '-u' "elastic:${ELASTIC_PASSWORD}" )
+  fi
+
+  local -i result=1
+  local output
+
+  output="$(curl "${args[@]}")"
+  if [[ "${output: -3}" -eq 200 ]]; then
+    result=0
+  fi
+
+  if ((result)); then
+    echo -e "\n${output::-3}\n"
+  fi
+
   return $result
 }

infrastructure/elasticsearch/setup-settings.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# OpenCRVS is also distributed under the terms of the Civil Registration
+# & Healthcare Disclaimer located at http://opencrvs.org/license.
+#
+# Copyright (C) The OpenCRVS Authors. OpenCRVS and the OpenCRVS
+# graphic logo are (registered/a) trademark(s) of Plan International.
+
+set -eu
+set -o pipefail
+
+source "$(dirname "${BASH_SOURCE[0]}")/setup-helpers.sh"
+
+echo "-------- $(date) --------"
+
+log 'Waiting for availability of Elasticsearch'
+wait_for_elasticsearch
+
+ensure_settings "{\"index\":{\"number_of_replicas\":0}}"

infrastructure/elasticsearch/setup-users.sh

@@ -13,9 +13,6 @@
 set -eu
 set -o pipefail
 
-apt-get update
-apt-get install curl -y
-
 source "$(dirname "${BASH_SOURCE[0]}")/setup-helpers.sh"
 
 

infrastructure/elasticsearch/setup.sh

@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+#
+# OpenCRVS is also distributed under the terms of the Civil Registration
+# & Healthcare Disclaimer located at http://opencrvs.org/license.
+#
+# Copyright (C) The OpenCRVS Authors. OpenCRVS and the OpenCRVS
+# graphic logo are (registered/a) trademark(s) of Plan International.
+
+set -eu
+set -o pipefail
+
+apt-get update
+apt-get install curl -y
+
+bash "$(dirname "${BASH_SOURCE[0]}")/setup-users.sh"
+bash "$(dirname "${BASH_SOURCE[0]}")/setup-settings.sh"
+
+
+

infrastructure/monitoring/kibana/config.ndjson

@@ -54,6 +54,7 @@ monitoring.ui.container.elasticsearch.enabled: true
 xpack.encryptedSavedObjects.encryptionKey: '{{KIBANA_ENCRYPTION_KEY}}'
 xpack.reporting.encryptionKey: '{{KIBANA_ENCRYPTION_KEY}}'
 xpack.actions.preconfiguredAlertHistoryEsIndex: true
+xpack.infra.sources.default.logAlias: 'logs-*,filebeat-*,kibana_sample_data_logs*,logstash*'
 # If your Elasticsearch is protected with basic authentication, these settings provide
 # the username and password that the Kibana server uses to perform maintenance on the Kibana
 # index at startup. Your Kibana users still need to authenticate with Elasticsearch, which