diff --git a/ansible/roles/elastic_beats/tasks/main.yml b/ansible/roles/elastic_beats/tasks/main.yml index 2646f6e0..98dba759 100644 --- a/ansible/roles/elastic_beats/tasks/main.yml +++ b/ansible/roles/elastic_beats/tasks/main.yml @@ -110,6 +110,12 @@ hosts: ["unix:///var/run/docker.sock"] period: 10s enabled: true + + - module: prometheus + period: 10s + hosts: ["{{ private_ip }}:{{ prometheus_port }}"] + metrics_path: "/metrics" + namespace: "drive_tenderdash" output_conf: elasticsearch: hosts: diff --git a/ansible/roles/elastic_beats/vars/tenderdash.yml b/ansible/roles/elastic_beats/vars/tenderdash.yml index f958ec10..003fa7ea 100644 --- a/ansible/roles/elastic_beats/vars/tenderdash.yml +++ b/ansible/roles/elastic_beats/vars/tenderdash.yml @@ -22,3 +22,10 @@ platform_filebeat_inputs: fields: - from: "json.level" to: "log.level" + +metricbeat_modules: + - module: prometheus + period: 10s + hosts: ["{{ private_ip }}:{{ prometheus_port }}"] + metrics_path: "/metrics" + namespace: "drive_tenderdash" diff --git a/ansible/roles/elastic_stack/files/pipelines/process_proposal_pipeline.json b/ansible/roles/elastic_stack/files/pipelines/process_proposal_pipeline.json new file mode 100644 index 00000000..7a0261f0 --- /dev/null +++ b/ansible/roles/elastic_stack/files/pipelines/process_proposal_pipeline.json @@ -0,0 +1,11 @@ +{ + "description": "Parse round number from Tenderdash logs", + "processors": [ + { + "grok": { + "field": "message", + "patterns": ["Processed proposal with .* for height: %{NUMBER:height}, round: %{NUMBER:round}"] + } + } + ] + } \ No newline at end of file diff --git a/ansible/roles/elastic_stack/files/watches/error_logs_watch.json b/ansible/roles/elastic_stack/files/watches/error_logs_watch.json new file mode 100644 index 00000000..ac47c449 --- /dev/null +++ b/ansible/roles/elastic_stack/files/watches/error_logs_watch.json @@ -0,0 +1,33 @@ +{ + "trigger": { + "schedule": { + "interval": "10s" + } + }, + "input": { + "search": { + "request": { + "indices": ["logs-*"], + "body": { + "query": { + "bool": { + "should": [ + { "match": { "log.level": "ERROR" }}, + { "match": { "log.level": "FATAL" }} + ], + "minimum_should_match": 1 + } + } + } + } + } + }, + "actions": { + "log_alert": { + "logging": { + "text": "Alert triggered: {{ctx.payload}}" + } + } + } + } + \ No newline at end of file diff --git a/ansible/roles/elastic_stack/files/watches/high_round_watch.json b/ansible/roles/elastic_stack/files/watches/high_round_watch.json new file mode 100644 index 00000000..d6c96508 --- /dev/null +++ b/ansible/roles/elastic_stack/files/watches/high_round_watch.json @@ -0,0 +1,41 @@ +{ + "trigger": { + "schedule": { + "interval": "10s" + } + }, + "input": { + "search": { + "request": { + "indices": ["logs-*"], + "body": { + "query": { + "bool": { + "must": [ + { + "match": { + "message": "Processed proposal" + } + }, + { + "range": { + "round": { + "gt": 5 + } + } + } + ] + } + } + } + } + } + }, + "actions": { + "log_alert": { + "logging": { + "text": "Alert triggered: {{ctx.payload}}" + } + } + } + } \ No newline at end of file diff --git a/ansible/roles/elastic_stack/files/watches/no_logs_watch.json b/ansible/roles/elastic_stack/files/watches/no_logs_watch.json new file mode 100644 index 00000000..c83d8c3b --- /dev/null +++ b/ansible/roles/elastic_stack/files/watches/no_logs_watch.json @@ -0,0 +1,46 @@ +{ + "trigger": { + "schedule": { + "interval": "5m" + } + }, + "input": { + "search": { + "request": { + "indices": ["logs-*"], + "body": { + "query": { + "bool": { + "must_not": { + "exists": { + "field": "message" + } + }, + "filter": { + "range": { + "@timestamp": { + "gte": "now-5m" + } + } + } + } + } + } + } + } + }, + "condition": { + "compare": { + "ctx.payload.hits.total": { + "eq": 0 + } + } + }, + "actions": { + "log_alert": { + "logging": { + "text": "Alert triggered: {{ctx.payload}}" + } + } + } + } \ No newline at end of file diff --git a/ansible/roles/elastic_stack/tasks/main.yml b/ansible/roles/elastic_stack/tasks/main.yml index 2a5b955a..bfce2e18 100644 --- a/ansible/roles/elastic_stack/tasks/main.yml +++ b/ansible/roles/elastic_stack/tasks/main.yml @@ -119,3 +119,30 @@ ansible.builtin.import_tasks: configure_cluster.yml run_once: true delegate_to: '{{ play_hosts | first }}' + + +- name: Upload and apply Tenderdash log parsing pipeline + ansible.builtin.uri: + url: "{{ elasticsearch_url }}/logstash/_pipeline/process_proposal_pipeline" + method: PUT + body: "{{ lookup('file', 'files/pipelines/process_proposal_pipeline.json') }}" + body_format: json + user: "{{ elastic_username }}" + password: "{{ elastic_password }}" + when: inventory_hostname in groups['elasticsearch'] + +- name: Create Elasticsearch Watches + ansible.builtin.uri: + url: "{{ elasticsearch_url }}/_watcher/watch/{{ item.name }}" + method: PUT + body: "{{ lookup('file', item.file) }}" + body_format: json + user: "{{ elastic_username }}" + password: "{{ elastic_password }}" + loop: + - { name: 'error_logs_watch', file: 'files/watches/error_logs_watch.json' } + - { name: 'high_round_watch', file: 'files/watches/high_round_watch.json' } + - { name: 'no_logs_watch', file: 'files/watches/no_logs_watch.json' } + loop_control: + label: "{{ item.name }}" + when: inventory_hostname in groups['elasticsearch'] diff --git a/ansible/roles/elastic_stack/vars/main.yml b/ansible/roles/elastic_stack/vars/main.yml index f99a9d3e..db525ba3 100644 --- a/ansible/roles/elastic_stack/vars/main.yml +++ b/ansible/roles/elastic_stack/vars/main.yml @@ -4,3 +4,4 @@ bundle_path: '{{ elastic_path }}/bundle' certs_path: '{{ elastic_path }}/certificates' data_path: '{{ elastic_path }}/data' kibana_port: 5601 +elasticsearch_url: "http://{{ hostvars[inventory_hostname].private_ip }}:9200"