441441# 'Optional (defaults to False). If true, stdout/stderr logs will contain docker timestamps at the beginning of the line\n',
442442# convert_to=bool, default=False)
443443
444+ define_config_option (
445+ __monitor__ ,
446+ "k8s_label_include_globs" ,
447+ "Optional, (defaults to ['*']). Specifies a list of K8s labels to be added to logs." ,
448+ convert_to = ArrayOfStrings ,
449+ default = ["*" ],
450+ env_aware = True ,
451+ )
452+
453+ define_config_option (
454+ __monitor__ ,
455+ "k8s_label_exclude_globs" ,
456+ "Optional, (defaults to []]). Specifies a list of K8s labels to be ignored and not added to logs." ,
457+ convert_to = ArrayOfStrings ,
458+ default = [],
459+ env_aware = True ,
460+ )
461+
444462define_metric (
445463 __monitor__ ,
446464 "docker.net.rx_bytes" ,
@@ -3232,6 +3250,15 @@ def __get_base_attributes(self):
32323250
32333251 return attributes
32343252
3253+ def __is_label_allowed (self , label_name ):
3254+ return any (
3255+ fnmatch .fnmatch (label_name , glob )
3256+ for glob in self ._config .get ("k8s_label_include_globs" )
3257+ ) and not any (
3258+ fnmatch .fnmatch (label_name , glob )
3259+ for glob in self ._config .get ("k8s_label_exclude_globs" )
3260+ )
3261+
32353262 def __get_log_config_for_container (self , cid , info , k8s_cache , base_attributes ):
32363263 # type: (str, dict, KubernetesCache, JsonObject) -> List[Dict]
32373264
@@ -3300,7 +3327,8 @@ def __get_log_config_for_container(self, cid, info, k8s_cache, base_attributes):
33003327 container_attributes ["k8s_node" ] = pod .node_name
33013328
33023329 for label , value in six .iteritems (pod .labels ):
3303- container_attributes [label ] = value
3330+ if self .__is_label_allowed (label ):
3331+ container_attributes [label ] = value
33043332
33053333 if "parser" in pod .labels :
33063334 parser = pod .labels ["parser" ]
@@ -3321,7 +3349,11 @@ def __get_log_config_for_container(self, cid, info, k8s_cache, base_attributes):
33213349 # field `_k8s_ck`
33223350 container_attributes ["_k8s_dn" ] = controller .name
33233351 # `_k8s_dl` is translated to `k8s-labels`
3324- container_attributes ["_k8s_dl" ] = controller .flat_labels
3352+ container_attributes ["_k8s_dl" ] = "," .join (
3353+ f"{ label } { value }
3354+ for label , value in controller .labels .items ()
3355+ if self .__is_label_allowed (label )
3356+ )
33253357 # `_k8s_ck` is translated into the field key for
33263358 # `_k8s_dn`. Here are some examples: `k8s-deployment`,
33273359 # `k8s-daemon-set`, `k8s-stateful-set`, etc. If the
@@ -3639,6 +3671,232 @@ class KubernetesMonitor(
36393671* lineGroupers (not supported at all)
36403672* path (the path is always fixed for k8s container logs)
36413673
3674+ ### Configuring multiple accounts per container
3675+
3676+ Besides the default API key stored in the scalyr/scalyr-api-key secret, the user can specify API keys for namespaces, pods and containers using annotations.
3677+
3678+ #### Namespace level API keys
3679+ ```log.config.scalyr.com/teams.{team_number}.secret: {secret_name}```
3680+
3681+ Overrides the default API key for all pods in the namespace.
3682+ The `secret_name` is the name of the secret (stored in the same namespace) holding the Scalyr API key.
3683+ The `teams_number` is an arbitrary unique number.
3684+
3685+ #### Pod level API keys
3686+ ```log.config.scalyr.com/teams.{team_number}.secret: {secret_name}```
3687+
3688+ Overrides the default API key and the namespace API key for all containers in the pod.
3689+ The `secret_name` is the name of the secret (stored in the same namespace as the pod) holding the Scalyr API key.
3690+ The `teams_number` is an arbitrary unique number.
3691+
3692+ #### Container level API keys
3693+ ```log.config.scalyr.com/{container_name}.teams.{team_number}.secret: {secret_name}```
3694+
3695+ Overrides the default API key, the namespace API key and the pod API keys for all containers in the pod.
3696+ The `secret_name` is the name of the secret (stored in the same namespace as the pod) holding the Scalyr API key.
3697+ The `teams_number` is an arbitrary unique number.
3698+
3699+ #### API Key Secret structure
3700+
3701+ ```yaml
3702+ apiVersion: v1
3703+ kind: Secret
3704+ data:
3705+ scalyr-api-key: <b64 encoded API key>
3706+ ```
3707+
3708+
3709+ #### Simple visual example of Secret key annotation priority
3710+
3711+ > [!NOTE]
3712+ > When no annotation is present for either the namespace or pod, the default secret _scalyr/scalyr-api-key_ is used.
3713+
3714+ 
3715+
3716+
3717+ #### Example:
3718+
3719+ #### Configuration:
3720+
3721+ ##### Default API key for the Scalyr Agent
3722+ ```yaml
3723+ apiVersion: v1
3724+ kind: Secret
3725+ metadata:
3726+ name: scalyr-api-key
3727+ namespace: scalyr
3728+ data:
3729+ scalyr-api-key: <b64 encoded SCALYR_API_KEY_WRITE_TEAM_1>
3730+ ```
3731+
3732+ ##### Workload Namespaces
3733+ ```yaml
3734+ apiVersion: v1
3735+ kind: Namespace
3736+ metadata:
3737+ name: workload-namespace-1
3738+ annotations:
3739+ log.config.scalyr.com/teams.1.secret: scalyr-api-key-team-2
3740+ ---
3741+ apiVersion: v1
3742+ kind: Namespace
3743+ metadata:
3744+ name: workload-namespace-2
3745+ ```
3746+
3747+ ##### API keys in the workload-namespace-1 Namespace
3748+ ```yaml
3749+ apiVersion: v1
3750+ kind: Secret
3751+ metadata:
3752+ name: scalyr-api-key-team-2
3753+ namespace: workload-namespace-1
3754+ data:
3755+ scalyr-api-key: <b64 encoded SCALYR_API_KEY_WRITE_TEAM_2>
3756+ ---
3757+ apiVersion: v1
3758+ kind: Secret
3759+ metadata:
3760+ name: scalyr-api-key-team-3
3761+ data:
3762+ scalyr-api-key: <b64 encoded SCALYR_API_KEY_WRITE_TEAM_3>
3763+ ---
3764+ apiVersion: v1
3765+ kind: Secret
3766+ metadata:
3767+ name: scalyr-api-key-team-4
3768+ data:
3769+ scalyr-api-key: <b64 encoded SCALYR_API_KEY_WRITE_TEAM_4>
3770+ ---
3771+ apiVersion: v1
3772+ kind: Secret
3773+ metadata:
3774+ name: scalyr-api-key-team-5
3775+ data:
3776+ scalyr-api-key: <b64 encoded SCALYR_API_KEY_WRITE_TEAM_5>
3777+ ---
3778+ apiVersion: v1
3779+ kind: Secret
3780+ metadata:
3781+ name: scalyr-api-key-team-6
3782+ data:
3783+ scalyr-api-key: <b64 encoded SCALYR_API_KEY_WRITE_TEAM_6>
3784+ ---
3785+ apiVersion: v1
3786+ kind: Secret
3787+ metadata:
3788+ name: scalyr-api-key-team-7
3789+ data:
3790+ scalyr-api-key: <b64 encoded SCALYR_API_KEY_WRITE_TEAM_7>
3791+ ```
3792+
3793+ #### Workload Pods in the workload-namespace Namespace
3794+ ```yaml
3795+ apiVersion: v1
3796+ kind: Pod
3797+ metadata:
3798+ labels:
3799+ app: multi-account-test
3800+ name: workload-pod-1
3801+ namespace: workload-namespace-1
3802+ annotations:
3803+ log.config.scalyr.com/teams.1.secret: "scalyr-api-key-team-3"
3804+ log.config.scalyr.com/teams.5.secret: "scalyr-api-key-team-4"
3805+ log.config.scalyr.com/workload-pod-1-container-1.teams.1.secret: "scalyr-api-key-team-5"
3806+ log.config.scalyr.com/workload-pod-1-container-2.teams.1.secret: "scalyr-api-key-team-6"
3807+ log.config.scalyr.com/workload-pod-1-container-2.teams.2.secret: "scalyr-api-key-team-7"
3808+ spec:
3809+ containers:
3810+ - name: workload-pod-1-container-1
3811+ image: busybox
3812+ command:
3813+ - /bin/sh
3814+ - -c
3815+ - while true; do echo workload-pod-1-container-1; sleep 1; done
3816+ - name: workload-pod-1-container-2
3817+ image: busybox
3818+ command:
3819+ - /bin/sh
3820+ - -c
3821+ - while true; do echo workload-pod-1-container-2; sleep 1; done
3822+ - name: workload-pod-1-container-3
3823+ image: busybox
3824+ command:
3825+ - /bin/sh
3826+ - -c
3827+ - while true; do echo workload-pod-1-container-3; sleep 1; done
3828+ ---
3829+ apiVersion: v1
3830+ kind: Pod
3831+ metadata:
3832+ labels:
3833+ app: multi-account-test
3834+ name: workload-pod-2
3835+ namespace: workload-namespace-1
3836+ spec:
3837+ containers:
3838+ - name: workload-pod-2-container-1
3839+ image: busybox
3840+ command:
3841+ - /bin/sh
3842+ - -c
3843+ - while true; do echo workload-pod-2-container-1; sleep 1; done
3844+ ```
3845+
3846+ ##### Workload Pod in workload-namespace-2 Namespace
3847+ ```yaml
3848+ apiVersion: v1
3849+ kind: Pod
3850+ metadata:
3851+ labels:
3852+ app: multi-account-test
3853+ name: workload-pod-3
3854+ namespace: workload-namespace-2
3855+ spec:
3856+ containers:
3857+ - name: workload-pod-3-container-1
3858+ image: busybox
3859+ command:
3860+ - /bin/sh
3861+ - -c
3862+ - while true; do echo workload-pod-3-container-1; sleep 1; done
3863+ ```
3864+
3865+ #### Ingested data:
3866+
3867+
3868+ | Container Name | API keys used to ingest logs | Note |
3869+ | --- |----------------------------------------------------------|-----------------------------|
3870+ | workload-pod-1-container-1 | SCALYR_API_KEY_WRITE_TEAM_4 | Container specific api keys |
3871+ | workload-pod-1-container-2 | SCALYR_API_KEY_WRITE_TEAM_5, SCALYR_API_KEY_WRITE_TEAM_6 | Container specific api keys |
3872+ | workload-pod-1-container-3 | SCALYR_API_KEY_WRITE_TEAM_3, SCALYR_API_KEY_WRITE_TEAM_4 | Pod default api keys |
3873+ | workload-pod-2-container-1 | SCALYR_API_KEY_WRITE_TEAM_2 | Namespace default api key |
3874+ | workload-pod-3-container-1 | SCALYR_API_KEY_WRITE_TEAM_1 | Agent default api key |
3875+
3876+
3877+ #### Querying the data:
3878+
3879+ ```bash
3880+ scalyr_readlog_token=SCALYR_API_KEY_WRITE_TEAM_1 scalyr query 'app="multi-account-test"' --columns=message
3881+ # workload-pod-3-container-1
3882+
3883+ scalyr_readlog_token=SCALYR_API_KEY_WRITE_TEAM_2 scalyr query 'app="multi-account-test"' --columns=message
3884+ # workload-pod-2-container-1
3885+
3886+ scalyr_readlog_token=SCALYR_API_KEY_WRITE_TEAM_3 scalyr query 'app="multi-account-test"' --columns=message
3887+ # workload-pod-1-container-3
3888+
3889+ scalyr_readlog_token=SCALYR_API_KEY_WRITE_TEAM_4 scalyr query 'app="multi-account-test"' --columns=message
3890+ # workload-pod-1-container-3
3891+ # workload-pod-1-container-1
3892+
3893+ scalyr_readlog_token=SCALYR_API_KEY_WRITE_TEAM_5 scalyr query 'app="multi-account-test"' --columns=message
3894+ # workload-pod-1-container-2
3895+
3896+ scalyr_readlog_token=SCALYR_API_KEY_WRITE_TEAM_6 scalyr query 'app="multi-account-test"' --columns=message
3897+ # workload-pod-1-container-2
3898+ ```
3899+
36423900### Excluding Logs
36433901
36443902Containers and pods can be specifically included/excluded from having their logs collected and
@@ -3653,6 +3911,8 @@ class KubernetesMonitor(
36533911
36543912 log.config.scalyr.com/include: false
36553913
3914+ In an edge case when a short-lived container metadata is not available anymore via K8s API and some logs are found, they will be collected based on `k8s_include_all_containers` flag.
3915+
36563916By default the agent monitors the logs of all pods/containers, and you have to manually exclude
36573917pods/containers you don't want. You can also set `k8s_include_all_containers: false` in the
36583918kubernetes_monitor monitor config section of `agent.d/docker.json`, in which case all containers are
0 commit comments