Merge pull request #134 from pluralsh/better-logstash

Better logstash handling

Author: michaeljguarino

services/logstash/beat.yaml.liquid

@@ -17,11 +17,17 @@ spec:
               type: container
               paths:
               - /var/log/containers/*${data.kubernetes.container.id}.log
+              ignore_older: 24h
     output.logstash:
       hosts: ["logstash-ls-beats.elastic:5044"]
     processors:
     - add_cloud_metadata: {}
     - add_host_metadata: {}
+    - add_kubernetes_metadata:
+        node: ${NODE_NAME}
+        add_resource_metadata:
+          deployment: true
+          cronjob: true
     - add_fields:
         target: cluster
         fields:

services/logstash/logstash.yaml.liquid

@@ -17,13 +17,20 @@ spec:
           cpu: 500m
   pipelines:
     - pipeline.id: main
+      # Defaults are 125 / 50ms; raise both to batch more events per cycle (fewer ES bulks, slightly higher latency).
+      pipeline.batch.size: 200
+      pipeline.batch.delay: 150
       config.string: |
         input {
           beats {
             port => 5044
           }
         }
         filter {
+          age {}
+          if [@metadata][age] > 86400 {
+            drop {}
+          }
           grok {
             match => { "message" => "%{HTTPD_COMMONLOG}"}
           }
@@ -39,6 +46,9 @@ spec:
             password => "${UPSTREAM_ES_PASSWORD}"
             index => "{{ contexts["plrl/cloud/observability"].elastic.user }}-logs-%{+YYYY.MM.dd}"
             manage_template => false
+            # Longer timeout while ES is under merge/ILM load (default 60s).
+            timeout => 120
+            retry_max_interval => 120
           }
         }
   services: