feat: metric and load test nodes (#607)

* feat: load test nodes

* fix: increase soft ulimit as well

* feat: metrics provisioning

* feat: deploy load test nodes with ansible

* fix: ansible host to run on

* chore: default to zero for LT nodes

* chore: instance type change

* feat: initial prometheus implementation in tf

* refactor: rename prometheus to metrics nodes

* chore: accomplish metrics and load test roles

* chore: LT nodes should be AMD

* chore: temp fixes

* fix: duplicate prometheus port

* chore: install latest protoc

* fix: invalid .env

* chore: redo insight API url

* fix: change to http

* fix: use dash_devnet_name var instead

* chore: restore deploy platbook

* modified_network_configs

* chore: configure tenderdash mempool cache size

* chore: update ouzo

* chore: uncomment code

* chore: added main_domain var to ansible initial config

* chore: configure tenderdash mempool cache size

* chore: update ouzo

* chore: uncomment code

* chore: linter fixes

* chore: stop complaining linter

* chore: set metrics/lt to always 1

* chore: fix CI

* chore: more CI fixing

* chore: final fix

* chore: enable recheck

* clean up

* chore: remove mixerCount

* feat: dynamic update of yml (#626)

* feat: dynamic update of yml

* feat: if less than

* chore: fix debug + new line

---------

Co-authored-by: ktechmidas <[email protected]>

* chore: remove mixer/LT from generate

* fix: linter

* fix: linter

* fix: more linter

* fix: even more linting

* fix: auto lint fix

* fix: final lint fixing

---------

Co-authored-by: ktechmidas <[email protected]>
Co-authored-by: vivekgsharma <[email protected]>

Author: 3 people

ansible/deploy.yml

@@ -77,6 +77,19 @@
   roles:
     - elastic_stack
 
+- name: Setup load tester
+  hosts: load_test
+  become: true
+  roles:
+    - role: protobuf_compiler
+    - role: load_tool
+
+- name: Setup metrics
+  hosts: metrics
+  become: true
+  roles:
+    - role: metrics
+
 - name: Set up miners
   hosts: miners
   become: true
@@ -231,6 +244,21 @@
       when: dash_network != "mainnet" and dash_network != "testnet"
       delegate_to: '{{ play_hosts | first }}'
 
+
+- name: Fund load tester nodes
+  hosts: wallet_nodes
+  become: true
+  tasks:
+    - name: Extract load tester addresses
+      ansible.builtin.set_fact:
+        load_tester_addresses: "{{ load_testers.values() | map(attribute='wallet.address') | list }}"
+
+    - name: Include funding for load testers
+      ansible.builtin.include_tasks: ./roles/mn_fund_collateral/tasks/fund_collateral.yml
+      vars:
+        payment_targets: "{{ load_tester_addresses }}"
+        amount: "{{ load_tester_wallet_amount }}"
+
 # todo: partially working code causes errors in deploy, comment out for now and fix later
 # - name: Create governance proposals
 #   hosts: wallet_nodes

ansible/group_vars/all

@@ -174,3 +174,8 @@ masternode_network_latency_max: 100
 # Dashmate stuff
 dashmate_home: /home/dashmate
 dashmate_logs_dir: '{{ dashmate_home }}/logs'
+
+# Wallet funding amount for load testing nodes
+load_tester_wallet_amount: 100
+
+prometheus_port: 36660

ansible/roles/dashmate/defaults/main.yml

@@ -23,6 +23,7 @@ dashmate_platform_drive_tenderdash_p2p_max_packet_msg_payload_size: 10240
 dashmate_platform_drive_tenderdash_p2p_send_rate: 5120000
 dashmate_platform_drive_tenderdash_p2p_recv_rate: 5120000
 dashmate_platform_drive_tenderdash_mempool_size: 5000
+dashmate_platform_drive_tenderdash_mempool_cache_size: 10000
 dashmate_platform_drive_tenderdash_mempool_max_txs_bytes: 536870912 # 500Mb, default: 1073741824
 dashmate_platform_drive_tenderdash_rpc_max_open_connections: 900
 dashmate_platform_drive_tenderdash_consensus_peer_gossip_sleep_duration: "100ms"

ansible/roles/dashmate/tasks/main.yml

@@ -144,9 +144,8 @@
     mode: "0644"
   register: template_result
 
-# We need to modify any config option to force dashmate render service configs
 - name: Render service configs
-  ansible.builtin.command: "{{ dashmate_cmd }} config set dashmate.helper.api.enable true"
+  ansible.builtin.command: "{{ dashmate_cmd }} config render"
   become: true
   become_user: dashmate
   args:

ansible/roles/dashmate/templates/dashmate.json.j2

@@ -213,6 +213,7 @@
               "recvRate": {{dashmate_platform_drive_tenderdash_p2p_recv_rate}}
             },
             "mempool": {
+              "cacheSize": {{dashmate_platform_drive_tenderdash_mempool_cache_size}},
               "size": {{dashmate_platform_drive_tenderdash_mempool_size}},
               "maxTxsBytes": {{dashmate_platform_drive_tenderdash_mempool_max_txs_bytes}}
             },
@@ -228,7 +229,7 @@
             "metrics": {
               "enabled": true,
               "host": "{{ private_ip }}",
-              "port": 36660
+              "port": {{ prometheus_port }}
             },
             "consensus": {
               "createEmptyBlocks": true,

ansible/roles/load_tool/tasks/main.yml

@@ -0,0 +1,63 @@
+---
+
+- name: Increase nofile limit
+  community.general.pam_limits:
+    domain: '*'
+    limit_type: "-"
+    limit_item: nofile
+    value: 60240
+
+- name: Install system dependencies
+  ansible.builtin.package:
+    name:
+      - cmake
+      - clang
+      - build-essential
+      - libc-dev
+      - libssl-dev
+      - pkg-config
+    state: present
+  when: dashmate_platform_enable
+
+- name: Download Rustup
+  ansible.builtin.get_url:
+    url: https://sh.rustup.rs
+    dest: '/home/ubuntu/rustup.sh'
+    owner: 'ubuntu'
+    group: 'ubuntu'
+    mode: '0755'
+  register: rustup
+
+- name: Install Rust
+  ansible.builtin.command:
+    cmd: "/home/ubuntu/rustup.sh -y"
+  become: true
+  become_user: ubuntu
+  when: rustup.changed # noqa: no-handler
+  changed_when: true
+
+- name: Clone platform
+  ansible.builtin.git:
+    repo: 'https://github.com/dashpay/platform.git'
+    dest: /home/ubuntu/load-test/platform
+    version: '{{ dashmate_branch }}'
+  become: true
+  become_user: ubuntu
+
+- name: Clone explorer
+  ansible.builtin.git:
+    repo: 'https://github.com/dashpay/rs-platform-explorer.git'
+    dest: /home/ubuntu/load-test/rs-platform-explorer
+    version: load-test
+  become: true
+  become_user: ubuntu
+
+- name: Create .env file
+  ansible.builtin.template:
+    src: .env.j2
+    dest: /home/ubuntu/load-test/rs-platform-explorer/.env
+    owner: ubuntu
+    group: ubuntu
+    mode: "0644"
+  become: true
+  become_user: ubuntu

ansible/roles/load_tool/templates/.env.j2

@@ -0,0 +1,24 @@
+# Comma-separated list of full URLs to DAPI nodes for the explorer to connect to.
+EXPLORER_DAPI_ADDRESSES={%- for hp_name in groups["hp_masternodes"] +%}http://{{ hostvars[hp_name]["public_ip"] }}:1443{%- if not loop.last %},{% endif %}{% endfor %}
+
+# The hostname or IP address of Dash Core RPC interface.
+EXPLORER_CORE_HOST={{ hostvars[groups["wallet_nodes"][0]]["private_ip"] }}
+
+# EXPLORER_CORE_RPC_PORT
+# The port number on which Dash Core RPC is listening.
+EXPLORER_CORE_RPC_PORT={{ dashd_rpc_port }}
+
+# The username for authentication with Dash Core RPC
+EXPLORER_CORE_RPC_USER={{ dashd_rpc_user }}
+
+# The password for authentication with Dash Core RPC
+EXPLORER_CORE_RPC_PASSWORD={{ dashd_rpc_password }}
+
+# Testnet Insight API URL
+EXPLORER_INSIGHT_API_URL=http://insight.{{ dash_devnet_name }}.{{ main_domain }}:{{ insight_port }}/insight-api
+
+# Network name
+EXPLORER_NETWORK={{ dash_network }}
+
+# TOOD: Create and fund a wallet
+EXPLORER_WALLET_PRIVATE_KEY={{ load_testers[inventory_hostname]["wallet"]["private_key"] }}

ansible/roles/metrics/defaults/main.yml

@@ -0,0 +1,4 @@
+---
+
+metrics_user: 'ubuntu'
+metrics_path: '/home/{{ metrics_user }}/metrics'

ansible/roles/metrics/files/haproxy/haproxy.cfg

@@ -0,0 +1,30 @@
+global
+    log stdout format raw local0
+
+defaults
+    log global
+    option  httplog
+    option  dontlognull
+
+
+frontend http_front
+    id 1
+    bind *:80
+    mode http
+
+    acl prometheus_path path_beg /prometheus
+    acl grafana_path path_beg /grafana
+
+
+    use_backend prometheus if prometheus_path
+    use_backend grafana if grafana_path
+
+backend prometheus
+#    http-request set-path %[path,regsub(^/prometheus,,g)]
+    server prometheus_server prometheus:9090
+    mode http
+
+backend grafana
+#    http-request set-path %[path,regsub(^/grafana,,g)]
+    server grafana_server grafana:3000
+    mode http

ansible/roles/metrics/files/prometheus/rules.yml

@@ -0,0 +1,21 @@
+---
+groups:
+  - name: Tenderdash chain
+    rules:
+      - alert: ChainHalt
+        expr: rate(drive_tenderdash_consensus_latest_block_height[10m]) == 0
+        for: 10m
+        labels:
+          severity: major
+          service: tenderdash
+        annotations:
+          description: 'Chain appears to be halted'
+
+      - alert: RoundTooHigh
+        expr: 'drive_tenderdash_consensus_rounds > 3'
+        for: 1m
+        labels:
+          severity: minor
+          service: tenderdash
+        annotations:
+          description: 'Voting round too high'

ansible/roles/metrics/tasks/main.yml

@@ -0,0 +1,47 @@
+---
+
+- name: Create a directories for the project
+  ansible.builtin.file:
+    path: '{{ metrics_path }}/{{ item }}'
+    state: directory
+    recurse: true
+    owner: '{{ metrics_user }}'
+    group: '{{ metrics_user }}'
+    mode: '0755'
+  loop:
+    - 'prometheus/config'
+    - 'grafana'
+    - 'haproxy'
+
+- name: Copy haproxy configs and prometheus rules
+  ansible.builtin.copy:
+    src: '{{ item }}'
+    dest: '{{ metrics_path }}/{{ item }}'
+    owner: '{{ metrics_user }}'
+    group: '{{ metrics_user }}'
+    mode: '0644'
+  loop:
+    - haproxy/haproxy.cfg
+    - prometheus/rules.yml
+
+- name: Render docker compose configs
+  ansible.builtin.template:
+    src: 'docker-compose.yml.j2'
+    dest: '{{ metrics_path }}/docker-compose.yml'
+    owner: '{{ metrics_user }}'
+    group: '{{ metrics_user }}'
+    mode: '0644'
+
+- name: Render prometheus configs
+  ansible.builtin.template:
+    src: 'prometheus.yml.j2'
+    dest: '{{ metrics_path }}/prometheus/config/prometheus.yml'
+    owner: '{{ metrics_user }}'
+    group: '{{ metrics_user }}'
+    mode: '0644'
+
+- name: Start metrics
+  community.docker.docker_compose:
+    project_src: '{{ metrics_path }}'
+    restarted: true
+    state: present

ansible/roles/metrics/templates/docker-compose.yml.j2

@@ -0,0 +1,33 @@
+  version: '3'
+  services:
+    haproxy:
+      image: haproxy:latest
+      ports:
+        - "80:80"
+      volumes:
+        - ./haproxy/haproxy.cfg:/usr/local/etc/haproxy/haproxy.cfg
+
+    prometheus:
+      user: root
+      image: prom/prometheus:latest
+      command:
+        - '--config.file=/etc/prometheus/prometheus.yml'
+        - '--web.external-url=http://{{ public_ip }}/prometheus'
+        - '--log.level=debug'
+      volumes:
+        - ./prometheus/config:/etc/prometheus
+        - /mnt/prometheus:/prometheus
+      expose:
+        - 9090
+
+    grafana:
+      image: grafana/grafana:latest
+      user: root 
+      volumes:
+        - ./grafana/data:/var/lib/grafana # TODO: Must be mounted on a volume or mnt?
+      environment:
+        - GF_SERVER_ROOT_URL=/grafana
+        - GF_SERVER_SERVE_FROM_SUB_PATH=true
+        - GF_SERVER_ROUTER_LOGGING=true
+      expose:
+        - 3000

ansible/roles/metrics/templates/prometheus.yml.j2

@@ -0,0 +1,44 @@
+---
+# my global config
+global:
+  scrape_interval: 5s # Set the scrape interval to every 5 seconds. Default is every 1 minute.
+  evaluation_interval: 15s # Evaluate rules every 15 seconds. The default is every 1 minute.
+  # scrape_timeout is set to 3s; global default is 10s.
+  scrape_timeout: 3s
+
+# Alertmanager configuration
+alerting:
+  alertmanagers:
+    - static_configs:
+        - targets:
+          # - alertmanager:9093
+
+# Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
+rule_files:
+  - "rules.yml"
+  # - "first_rules.yml"
+  # - "second_rules.yml"
+
+# A scrape configuration containing exactly one endpoint to scrape:
+# Here it's Prometheus itself.
+scrape_configs:
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: "prometheus"
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+    metrics_path: /prometheus/metrics
+    static_configs:
+      - targets: ["localhost:9090"]
+
+  # The job name is added as a label `job=<job_name>` to any timeseries scraped from this config.
+  - job_name: "tenderdash"
+
+    # metrics_path defaults to '/metrics'
+    # scheme defaults to 'http'.
+
+    static_configs:
+{% for hp_name in groups["hp_masternodes"] %}
+      - targets: ["{{ hostvars[hp_name]['private_ip'] }}:{{ prometheus_port }}"]
+        labels:
+          node: "{{ hp_name }}"
+{% endfor %}

ansible/roles/protobuf_compiler/defaults/main.yml

@@ -0,0 +1,6 @@
+protoc_version: "25.2"
+protoc_dest_path: "/opt/protoc"
+protoc_bin_path: "/usr/bin/protoc"
+protoc_zip_file_name: "protoc-{{ protoc_version }}-linux-{{ 'aarch_64' if ansible_architecture == 'aarch64' else 'x86_64' }}.zip"
+protoc_zip_url: "https://github.com/protocolbuffers/protobuf/releases/download/v{{ protoc_version }}/{{ protoc_zip_file_name }}"
+protoc_zip_path: "/tmp"

ansible/roles/protobuf_compiler/tasks/main.yml

@@ -0,0 +1,27 @@
+- name: Download protoc zip file
+  ansible.builtin.get_url:
+    url: "{{ protoc_zip_url }}"
+    dest: "/tmp/"
+    mode: '0644'
+
+- name: Ensure destination directory exists
+  ansible.builtin.file:
+    path: "{{ protoc_dest_path }}"
+    state: directory
+    mode: '0755'
+
+- name: Unzip protoc to the destination
+  ansible.builtin.unarchive:
+    src: "{{ protoc_zip_path }}/{{ protoc_zip_file_name }}"
+    dest: "{{ protoc_dest_path }}"
+    remote_src: true
+# TODO: Create an empty file with version and then check if it's exists
+#    creates: "{{ protoc_dest_path }}/{{ protoc_version }}-installed"
+    mode: '755'
+
+- name: Create symlink for protoc
+  ansible.builtin.file:
+    src: "{{ protoc_dest_path }}/bin/protoc"
+    dest: "{{ protoc_bin_path }}"
+    state: link
+    force: true