Fix k8s container logs input id using container names

Author: pchila

deploy/helm/elastic-agent/examples/eck/rendered/manifest.yaml

@@ -321,13 +321,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:

deploy/helm/elastic-agent/examples/kubernetes-default/rendered/manifest.yaml

@@ -320,13 +320,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -888,7 +888,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 233affcd72143e637a130b5f099c30e194d90042eb00a26512f51c844c65a821
+        checksum/config: 0a02c8b3783f5e072f6eb62d87c3fefb2df9cdaf8efb7dbda9bcc11050246c3d
     spec:
       automountServiceAccountToken: true
       containers:

deploy/helm/elastic-agent/examples/kubernetes-hints-autodiscover/rendered/manifest.yaml

@@ -320,13 +320,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -891,7 +891,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: daca0d998edb3afa587d96e69b0833f6919ca6ba72f58f3a1f83b22d7e5ffaf6
+        checksum/config: 440922798e9423e250298ce729018751f6a9d3343a03f3e54f93380e932acdfe
     spec:
       automountServiceAccountToken: true
       containers:

deploy/helm/elastic-agent/examples/kubernetes-ksm-sharding/rendered/manifest.yaml

@@ -132,13 +132,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -949,7 +949,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 05797fdfdd3cdeefb99e39e0f4756a6b812465509b31195ff57ae3925aa5e087
+        checksum/config: 916336da56b9ea407c8b613a721335c42af4fb6df724f54d0f301e6c193b8e4e
     spec:
       automountServiceAccountToken: true
       containers:

deploy/helm/elastic-agent/examples/kubernetes-only-logs/rendered/manifest.yaml

@@ -49,13 +49,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -223,7 +223,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 0840dcdf026f64cefb7aa69f420bc923d7e2d7d6e9a239e107fd2684e309d8ae
+        checksum/config: 7c35f814b93d366f98c472a618bd968083df7c07072b6e510cbb71ad26b0b6ab
     spec:
       automountServiceAccountToken: true
       containers:

deploy/helm/elastic-agent/examples/multiple-integrations/rendered/manifest.yaml

@@ -346,13 +346,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -917,7 +917,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: daca0d998edb3afa587d96e69b0833f6919ca6ba72f58f3a1f83b22d7e5ffaf6
+        checksum/config: 440922798e9423e250298ce729018751f6a9d3343a03f3e54f93380e932acdfe
     spec:
       automountServiceAccountToken: true
       containers:

deploy/helm/elastic-agent/examples/user-service-account/rendered/manifest.yaml

@@ -296,13 +296,13 @@ stringData:
         - data_stream:
             dataset: kubernetes.container_logs
             type: logs
-          id: kubernetes-container-logs-${kubernetes.pod.name}-${kubernetes.container.id}
+          id: kubernetes-container-logs-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
           parsers:
           - container:
               format: auto
               stream: all
           paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
           processors:
           - add_fields:
               fields:
@@ -868,7 +868,7 @@ spec:
       labels:
         name: agent-pernode-example
       annotations:
-        checksum/config: 233affcd72143e637a130b5f099c30e194d90042eb00a26512f51c844c65a821
+        checksum/config: 0a02c8b3783f5e072f6eb62d87c3fefb2df9cdaf8efb7dbda9bcc11050246c3d
     spec:
       automountServiceAccountToken: true
       containers:

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-daemonset-configmap.yaml

@@ -354,7 +354,7 @@ data:
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
       # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
-      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -366,7 +366,7 @@ data:
         streams:
           # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
           # Each filestream data stream creates a separate instance of the Filebeat filestream input.
-          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id}
+          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
             data_stream:
               dataset: kubernetes.container_logs
               type: logs

deploy/kubernetes/elastic-agent-kustomize/default/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml

@@ -122,7 +122,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -146,7 +146,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default

deploy/kubernetes/elastic-agent-kustomize/ksm-autosharding/elastic-agent-standalone/base/elastic-agent-standalone-ksm-daemonset-configmap.yaml

@@ -122,7 +122,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -146,7 +146,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default

deploy/kubernetes/elastic-agent-standalone-kubernetes.yaml

@@ -351,7 +351,7 @@ data:
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
       # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -363,7 +363,7 @@ data:
         streams:
           # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
           # Each filestream data stream creates a separate instance of the Filebeat filestream input.
-          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.id}
+          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
             data_stream:
               dataset: kubernetes.container_logs
               type: logs

deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-daemonset-configmap.yaml

@@ -351,7 +351,7 @@ data:
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
       # Input ID allowing Elastic Agent to track the state of this input. Must be unique.
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -363,7 +363,7 @@ data:
         streams:
           # Stream ID for this data stream allowing Filebeat to track the state of the ingested files. Must be unique.
           # Each filestream data stream creates a separate instance of the Filebeat filestream input.
-          - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+          - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
             data_stream:
               dataset: kubernetes.container_logs
               type: logs
@@ -378,7 +378,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default

deploy/kubernetes/elastic-agent-standalone/elastic-agent-standalone-ksm-daemonset-configmap.yaml

@@ -122,7 +122,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -146,7 +146,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default

dev-tools/kubernetes/base/elastic-agent-standalone/config-map.yaml

@@ -330,7 +330,7 @@ data:
               dataset: system.system
             condition: '${host.platform} == ''windows'''
             ignore_older: 72h
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         type: filestream
         use_output: default
         meta:
@@ -354,7 +354,7 @@ data:
               #     negate: true
               #     match: after
             paths:
-              - /var/log/containers/*${kubernetes.container.id}.log
+              - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
       - id: audit-log
         type: filestream
         use_output: default

internal/pkg/agent/application/coordinator/testdata/config.yaml

@@ -243,7 +243,7 @@ inputs:
           dataset: system.system
         condition: '${host.platform} == ''windows'''
         ignore_older: 72h
-  - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+  - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
     type: filestream
     use_output: default
     meta:
@@ -253,15 +253,15 @@ inputs:
     data_stream:
       namespace: default
     streams:
-      - id: container-log-${kubernetes.pod.name}-${kubernetes.container.id}
+      - id: container-log-${kubernetes.namespace}-${kubernetes.pod.name}-${kubernetes.container.name}
         data_stream:
           dataset: kubernetes.container_logs
           type: logs
         prospector.scanner.symlinks: true
         parsers:
           - container: ~
         paths:
-          - /var/log/containers/*${kubernetes.container.id}.log
+          - /var/log/pods/${kubernetes.namespace}_${kubernetes.pod.name}_${kubernetes.pod.uid}/${kubernetes.container.name}/*.log
   - id: audit-log
     type: filestream
     use_output: default