Switch k8s input paths to /var/log/pods/* to ingest rotated container logs

Author: pchila

deploy/helm/elastic-agent/templates/integrations/_kubernetes/_kubernetes_logs_containers.tpl

@@ -16,12 +16,12 @@ Config input for container logs
     namespace: {{ .Values.kubernetes.namespace }}
   use_output: {{ .Values.kubernetes.output }}
   streams:
-  - id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+  - id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
     data_stream:
       dataset: kubernetes.container_logs
       type: logs
     paths:
-      - '/var/log/containers/*${kubernetes.container.id}.log'
+      - '/var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log'
     prospector.scanner.symlinks: {{ dig "vars" "symlinks" true .Values.kubernetes.containers.logs }}
     parsers:
       - container:

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset-configmap.yaml

@@ -354,7 +354,7 @@ data:
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
       # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id}
         type: filestream
         use_output: default
         meta:
@@ -366,7 +366,7 @@ data:
         streams:
           # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
           # Each filestream data stream creates a separate instance of the Filebeat filestream input.
-          - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id}
             data_stream:
               dataset: kubernetes.container_logs
               type: logs
@@ -381,7 +381,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -363,7 +363,7 @@ data:
         streams:
           # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
           # Each filestream data stream creates a separate instance of the Filebeat filestream input.
-          - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id}
             data_stream:
               dataset: kubernetes.container_logs
               type: logs
@@ -378,7 +378,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default