feat: system and docker metrics in kibana (#617)

* feat: load test nodes

* fix: increase soft ulimit as well

* feat: metrics provisioning

* feat: deploy load test nodes with ansible

* fix: ansible host to run on

* chore: default to zero for LT nodes

* chore: instance type change

* feat: initial prometheus implementation in tf

* refactor: rename prometheus to metrics nodes

* chore: accomplish metrics and load test roles

* chore: LT nodes should be AMD

* chore: temp fixes

* fix: duplicate prometheus port

* chore: install latest protoc

* fix: invalid .env

* chore: redo insight API url

* fix: change to http

* fix: use dash_devnet_name var instead

* chore: restore deploy platbook

* modified_network_configs

* chore: configure tenderdash mempool cache size

* chore: update ouzo

* chore: uncomment code

* chore: added main_domain var to ansible initial config

* chore: configure tenderdash mempool cache size

* chore: update ouzo

* chore: uncomment code

* chore: linter fixes

* chore: stop complaining linter

* chore: set metrics/lt to always 1

* feat: system and docker metrics in kibana

* chore: commented code

* chore: update dashmate config

* Move 'Fund load tester nodes' block above 'Activate Sporks' in deploy.yml

* chore: fix CI

* clean up

* chore: update to platform 1.0-dev.9

* fix: linter

* fix: linter again

---------

Co-authored-by: ktechmidas <[email protected]>
Co-authored-by: vivekgsharma <[email protected]>
Co-authored-by: ktechmidas <[email protected]>

Author: 4 people

ansible/deploy.yml

@@ -236,14 +236,6 @@
     - role: mn_protx_config
       mnlist: "{{ hp_masternodes }}"
 
-- name: Activate sporks
-  hosts: wallet_nodes
-  become: true
-  roles:
-    - role: activate_dashd_sporks
-      when: dash_network != "mainnet" and dash_network != "testnet"
-      delegate_to: '{{ play_hosts | first }}'
-
 
 - name: Fund load tester nodes
   hosts: wallet_nodes
@@ -259,6 +251,15 @@
         payment_targets: "{{ load_tester_addresses }}"
         amount: "{{ load_tester_wallet_amount }}"
 
+
+- name: Activate sporks
+  hosts: wallet_nodes
+  become: true
+  roles:
+    - role: activate_dashd_sporks
+      when: dash_network != "mainnet" and dash_network != "testnet"
+      delegate_to: '{{ play_hosts | first }}'
+
 # todo: partially working code causes errors in deploy, comment out for now and fix later
 # - name: Create governance proposals
 #   hosts: wallet_nodes

ansible/requirements.yml

@@ -38,5 +38,8 @@ roles:
   - src: geerlingguy.pip
     version: 2.2.0
 
+  - src: elastic.beats
+    version: v7.17.0
+
   - src: ktechmidas.openvpn
     version: 1.0.0

ansible/roles/dashmate/defaults/main.yml

@@ -28,6 +28,11 @@ dashmate_platform_drive_tenderdash_mempool_max_txs_bytes: 536870912 # 500Mb, def
 dashmate_platform_drive_tenderdash_rpc_max_open_connections: 900
 dashmate_platform_drive_tenderdash_consensus_peer_gossip_sleep_duration: "100ms"
 dashmate_platform_drive_tenderdash_consensus_peer_query_maj23_sleep_duration: "2s"
+dashmate_platform_drive_tenderdash_mempool_timeout_check_tx: "1s"
+dashmate_platform_drive_tenderdash_mempool_tx_enqueue_timeout: "10ms"
+dashmate_platform_drive_tenderdash_mempool_tx_send_rate_limit: 10
+dashmate_platform_drive_tenderdash_mempool_tx_recv_rate_limit: 12
+dashmate_platform_drive_tenderdash_rpc_timeout_broadcast_tx: "1s"
 # Unsafe params. Use only if you understand what you are doing.
 dashmate_platform_drive_tenderdash_consensus_unsafe_override_propose_timeout:
 dashmate_platform_drive_tenderdash_consensus_unsafe_override_propose_delta:

ansible/roles/dashmate/templates/dashmate.json.j2

@@ -179,6 +179,12 @@
                 "color": null
               }{% endif %}
             },
+            "tokioConsole": {
+              "enabled": false,
+              "host": "127.0.0.1",
+              "port": 6669,
+              "retention_secs": 180
+            },
             "validatorSet": {
               "llmqType": {{ platform_drive_validator_set_llmq_type }}
             },
@@ -215,12 +221,17 @@
             "mempool": {
               "cacheSize": {{dashmate_platform_drive_tenderdash_mempool_cache_size}},
               "size": {{dashmate_platform_drive_tenderdash_mempool_size}},
-              "maxTxsBytes": {{dashmate_platform_drive_tenderdash_mempool_max_txs_bytes}}
+              "maxTxsBytes": {{dashmate_platform_drive_tenderdash_mempool_max_txs_bytes}},
+              "timeoutCheckTx": "{{dashmate_platform_drive_tenderdash_mempool_timeout_check_tx}}",
+              "txEnqueueTimeout": "{{dashmate_platform_drive_tenderdash_mempool_tx_enqueue_timeout}}",
+              "txSendRateLimit": {{dashmate_platform_drive_tenderdash_mempool_tx_send_rate_limit}},
+              "txRecvRateLimit": {{dashmate_platform_drive_tenderdash_mempool_tx_recv_rate_limit}}
             },
             "rpc": {
               "host": "127.0.0.1",
               "port": {{ tendermint_rpc_port }},
-              "maxOpenConnections": {{dashmate_platform_drive_tenderdash_rpc_max_open_connections}}
+              "maxOpenConnections": {{dashmate_platform_drive_tenderdash_rpc_max_open_connections}},
+              "timeoutBroadcastTx": "{{dashmate_platform_drive_tenderdash_rpc_timeout_broadcast_tx}}"
             },
             "pprof": {
               "enabled": {% if dashmate_platform_tenderdash_pprof_enable %}true{% else %}false{% endif %},

ansible/roles/elastic_beats/tasks/main.yml

@@ -57,3 +57,62 @@
 - name: Set up filebeat log monitoring
   ansible.builtin.include_role:
     name: geerlingguy.filebeat
+
+# TODO: Make sure we have retention policy for metrics
+- name: Set up metricbeat
+  ansible.builtin.include_role:
+    name: elastic.beats
+  vars:
+    beats_version: "{{ elastic_version }}"
+    beat: metricbeat
+    beat_conf:
+      setup:
+        dashboards:
+          enabled: true
+        kibana:
+          host: "{{ hostvars['logs-1'].private_ip }}:5601"
+          username: "{{ elastic_username }}"
+          password: "{{ elastic_password }}"
+      metricbeat:
+        modules:
+          - module: system
+            metricsets:
+              - cpu             # CPU usage
+              - load            # CPU load averages
+              - memory          # Memory usage
+              - network         # Network IO
+              - process         # Per process metrics
+              - process_summary # Process summary
+              - uptime          # System Uptime
+              - socket_summary  # Socket summary
+              - core           # Per CPU core usage
+              - diskio         # Disk IO
+              - fsstat         # File system summary metrics
+              - socket         # Sockets and connection info (linux only)
+            enabled: true
+            period: 10s
+            processes: ['.*']
+
+            # Configure the metric types that are included by these metricsets.
+            cpu.metrics: ["percentages", "normalized_percentages"]  # The other available option is ticks.
+            core.metrics: ["percentages"]  # The other available option is ticks.
+          - module: docker
+            metricsets:
+              - "container"
+              - "cpu"
+              - "diskio"
+              - "event"
+              - "healthcheck"
+              - "info"
+              - "memory"
+              - "network"
+              # - "network_summary"
+            hosts: ["unix:///var/run/docker.sock"]
+            period: 10s
+            enabled: true
+    output_conf:
+      elasticsearch:
+        hosts:
+          - "{{ hostvars['logs-1'].private_ip }}:9200"
+        username: "{{ elastic_username }}"
+        password: "{{ elastic_password }}"

lib/configGenerator/generateAnsibleConfig.js

@@ -216,7 +216,6 @@ async function generateAnsibleConfig(
   for (let i = 1; i <= hpMasternodesCount; i++) {
     config.hp_masternodes[`hp-masternode-${i}`] = await generateDip3Keys(network);
   }
-
   // Tenderdash keys
 
   function generateTenderdashNodeKeys() {