Support sending events to multiple Dataset accounts via K8s annotations (#1238)

* Support sending events to multiple Dataset accounts via K8s annotations

* Support sending events to multiple Dataset accounts via K8s annotations

* Script executable permission

* Addressing PR comments

* Addressing PR comments

* Addressing PR comments

* Addressing PR comments

* Addressing PR comments

* Addressing PR comments

Author: alesnovak-s1

.circleci/smoketest_docker.sh

@@ -145,6 +145,12 @@ echo "Uploader container ID == ${uploader_hostname}"
 echo "Using smoketest.py script from ${SMOKE_TESTS_SCRIPT_BRANCH} branch and URL ${SMOKE_TESTS_SCRIPT_URL}"
 
 function print_debugging_info_on_exit() {
+    echo ""
+    echo "::group::Scalyr Agent status"
+    echo ""
+    docker exec ${contname_agent} scalyr-agent-2 status -v || true
+    echo "::endgroup::"
+
     echo ""
     echo "::group::Docker logs for ${contname_agent} container"
     echo ""

.github/actions/install-k8s-agent/action.yml

@@ -9,6 +9,15 @@ inputs:
   scalyr_api_key:
     description: "Write API key to be used by the agent."
     required: true
+  scalyr_api_key_team_2:
+    description: "Write API key to be used by the agent."
+    required: false
+  scalyr_api_key_team_3:
+    description: "Write API key to be used by the agent."
+    required: false
+  scalyr_api_key_team_4:
+    description: "Write API key to be used by the agent."
+    required: false
   scalyr_cluster_name:
     description: "Cluster name to use."
     required: true
@@ -80,6 +89,21 @@ runs:
 
         # Define api key
         kubectl create secret generic scalyr-api-key --namespace scalyr --from-literal=scalyr-api-key="${{ inputs.scalyr_api_key }}"
+        
+        # Create a secret if the scalyr_api_key_team_2 is set
+        if [ ! -z "${{ inputs.scalyr_api_key_team_2 }}" ]; then
+          kubectl create secret generic scalyr-api-key-team-2 --namespace scalyr --from-literal=scalyr-api-key="${{ inputs.scalyr_api_key_team_2 }}"
+        fi
+        
+        # Create a secret if the scalyr_api_key_team_3 is set
+        if [ ! -z "${{ inputs.scalyr_api_key_team_3 }}" ]; then
+          kubectl create secret generic scalyr-api-key-team-3 --namespace scalyr --from-literal=scalyr-api-key="${{ inputs.scalyr_api_key_team_3 }}"
+        fi
+        
+        # Create a secret if the scalyr_api_key_team_4 is set
+        if [ ! -z "${{ inputs.scalyr_api_key_team_4 }}" ]; then
+          kubectl create secret generic scalyr-api-key-team-4 --namespace scalyr --from-literal=scalyr-api-key="${{ inputs.scalyr_api_key_team_4 }}"
+        fi
 
         # Create configmap
         kubectl create configmap --namespace scalyr scalyr-config \

.github/workflows/agent-build-new.yml

@@ -90,6 +90,12 @@ jobs:
       CT_AWS_DEV_EC2_SECRET_KEY: ${{ secrets.CT_AWS_DEV_EC2_SECRET_KEY }}
       CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
       CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}
+      SCALYR_API_KEY_READ_2: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_2_READ }}
+      SCALYR_API_KEY_READ_3: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_3_READ }}
+      SCALYR_API_KEY_READ_4: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_4_READ }}
+      SCALYR_API_KEY_WRITE_2: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_2_WRITE }}
+      SCALYR_API_KEY_WRITE_3: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_3_WRITE }}
+      SCALYR_API_KEY_WRITE_4: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_4_WRITE }}
       PULL_REGISTRY_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME_PROD_ACCOUNT }}
       PULL_REGISTRY_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD_PROD_ACCOUNT }}
       PUBLISH_REGISTRY_USERNAME: ${{ (github.ref_type == 'tag' || github.ref_name == 'master') && secrets.DOCKER_HUB_USERNAME_PROD_ACCOUNT || secrets.DOCKER_HUB_USERNAME_TEST_ACCOUNT }}

.github/workflows/reusable-agent-build-container-images.yml

@@ -44,6 +44,18 @@ on:
         required: true
       CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ:
         required: true
+      SCALYR_API_KEY_READ_2:
+        required: true
+      SCALYR_API_KEY_READ_3:
+        required: true
+      SCALYR_API_KEY_READ_4:
+        required: true
+      SCALYR_API_KEY_WRITE_2:
+        required: true
+      SCALYR_API_KEY_WRITE_3:
+        required: true
+      SCALYR_API_KEY_WRITE_4:
+        required: true
       PULL_REGISTRY_USERNAME:
         required: true
       PULL_REGISTRY_PASSWORD:
@@ -369,9 +381,11 @@ jobs:
           minikube image ls
           kubectl apply -f tests/e2e/k8s_k8s_monitor/std_printer_deployment.yaml
           kubectl apply -f tests/e2e/k8s_k8s_monitor/long_message_printer_deployment.yaml
+          kubectl apply -f tests/e2e/k8s_k8s_monitor/multiple_account_printers.yaml
           
           kubectl wait --for=condition=ready pod -l app=std-printer
           kubectl wait --for=condition=ready pod -l app=long-message-printer
+          kubectl wait --for=condition=ready pod -l app=multiple-account-printer
           kubectl get pods -A
 
           export APP_POD_NAME=$(kubectl get pod --namespace=default --selector=app=std-printer -o jsonpath="{.items[0].metadata.name}")
@@ -392,12 +406,17 @@ jobs:
           kubectl apply -f tests/e2e/k8s_events_monitor/cronjob_v1.yaml
 
           kubectl get cronjob -A
+      - name: Wait random time to make it easier for the cloud github runners
+        run: sleep $((RANDOM % 60))
 
       - name: Create scalyr-agent-2 daemonset
         uses: ./.github/actions/install-k8s-agent/
         with:
           scalyr_server: "agent.scalyr.com"
           scalyr_api_key: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}"
+          scalyr_api_key_team_2: "${{ secrets.SCALYR_API_KEY_WRITE_2 }}"
+          scalyr_api_key_team_3: "${{ secrets.SCALYR_API_KEY_WRITE_3 }}"
+          scalyr_api_key_team_4: "${{ secrets.SCALYR_API_KEY_WRITE_4 }}"
           scalyr_cluster_name: "${K8S_CLUSTER_NAME}"
           scalyr_k8s_events_disable: "false"
           main_yaml_path: "tests/e2e/scalyr-agent-2-daemonset.yaml"
@@ -520,6 +539,23 @@ jobs:
           echo "CronJob events checks"
           ./scripts/cicd/scalyr-query.sh '$serverHost="'${SCALYR_AGENT_POD_NAME}'" $logfile="/var/log/scalyr-agent-2/kubernetes_events.log" $parser="k8sEvents" k8s-kind="CronJob" involvedObjectKind="CronJob" involvedObjectName="hello" watchEventType="ADDED" reason="SawCompletedJob"'
 
+      - name: Verify multiaccount records have been ingested
+        timeout-minutes: 14
+        env:
+          scalyr_api_key_team: "${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_READ }}"
+          scalyr_api_key_team_2: "${{ secrets.SCALYR_API_KEY_READ_2 }}"
+          scalyr_api_key_team_3: "${{ secrets.SCALYR_API_KEY_READ_3 }}"
+          scalyr_api_key_team_4: "${{ secrets.SCALYR_API_KEY_READ_4 }}"
+          SCALYR_AGENT_POD_NAME: "${{ env.SCALYR_AGENT_POD_NAME }}"
+          ACCOUNT_NAME: "[email protected]"
+          ACCOUNT_NAME_2: "[email protected]"
+          ACCOUNT_NAME_3: "[email protected]"
+          ACCOUNT_NAME_4: "[email protected]"
+        run: |
+          echo 'Looking for multiple account logs'
+          
+          ./scripts/cicd/verify-multiaccount-records-have-been-ingested.sh
+
       - name: Notify Slack on Failure
         if: ${{ failure() && github.ref_name == 'master' }}
         uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
@@ -530,6 +566,7 @@ jobs:
           steps: ${{ toJson(steps) }}
           channel: '#eng-dataset-cloud-tech'
 
+
   k8s_open_metrics_monitor_tests:
     name: OpenMetrics Monitor - k8s ${{ inputs.builder_name }} ${{ matrix.k8s_version.version }}-${{ matrix.k8s_version.runtime}}
     runs-on: ubuntu-20.04

docs/monitors/kubernetes_monitor.md

@@ -30,6 +30,26 @@ fields behave differently when configured via k8s annotations:
 * lineGroupers (not supported at all)
 * path (the path is always fixed for k8s container logs)
 
+### Configuring multiple accounts per container
+
+Containers can be assigned one or more Scalyr API keys by specifying the `log.config.scalyr.com/{container_name}.teams.{team_number}.secret` annotation.  
+The `container_name` is the name of the container as specified in the pod spec, and the `team_number` is a unique number for each team.  
+The `secret` is the name of the secret holding the Scalyr API key for the team.  
+A default fallback api key can be specified by using the `log.config.scalyr.com/teams.{team_number}.secret` annotation.
+The monitor first looks into the container specific annotation, and if not found, it falls back to the default annotation or the global configuration api key.
+For example, if you had a pod with three containers `cont1`, `cont2` and `cont3` and you wanted to assign different Scalyr API keys to each container, 
+you could specify the following annotations:
+
+        log.config.scalyr.com/teams.1.secret: "scalyr-api-key-team-2"
+        log.config.scalyr.com/cont1.teams.2.secret: "scalyr-api-key-team-3"
+        log.config.scalyr.com/cont2.teams.1.secret: "scalyr-api-key-team-3"
+        log.config.scalyr.com/cont2.teams.2.secret: "scalyr-api-key-team-4"
+
+In this example
+* `cont1` would use the api key from `scalyr-api-key-team-3` secret
+* `cont2` would be sending the data to two teams accessible by api keys stored in `scalyr-api-key-team-3` and `scalyr-api-key-team-4` secrets.
+* `cont3` would use the default api key from `scalyr-api-key-team-2` secret
+
 ### Excluding Logs
 
 Containers and pods can be specifically included/excluded from having their logs collected and

k8s/no-kustomize/scalyr-service-account.yaml

@@ -12,7 +12,7 @@ metadata:
   name: scalyr-clusterrole
 rules:
 - apiGroups: [""]
-  resources: ["namespaces","pods","replicationcontrollers"]
+  resources: ["namespaces","pods","replicationcontrollers", "secrets"]
   verbs: ["get","list"]
 - apiGroups: [""]
   resources: ["nodes"]

scalyr_agent/agent_main.py

@@ -1877,6 +1877,7 @@ def __generate_status(self, warn_on_rate_limit=False):
         @rtype: AgentStatus
         """
         # Basic agent stats first.
+
         result = AgentStatus()
         result.launch_time = self.__start_time
         result.user = self.__controller.get_current_user()
@@ -1892,7 +1893,6 @@ def __generate_status(self, warn_on_rate_limit=False):
         # Describe the status of the configuration file.
         config_result = ConfigStatus()
         result.config_status = config_result
-
         config_result.last_check_time = self.__last_config_check_time
         if self.__current_bad_config is not None:
             config_result.path = self.__current_bad_config.file_path
@@ -2274,15 +2274,14 @@ def __report_status_to_file(self):
             # We do a little dance to write the status.  We write it to a temporary file first, and then
             # move it into the real location after the write has finished.  This way, the process watching
             # the file we are writing does not accidentally read it when it is only partially written.
+
             tmp_file_path = os.path.join(
                 self.__config.agent_data_path, "last_status.tmp"
             )
             final_file_path = os.path.join(self.__config.agent_data_path, "last_status")
-
             if os.path.isfile(final_file_path):
                 os.remove(final_file_path)
             tmp_file = open(tmp_file_path, "w")
-
             agent_status = self.__generate_status()
 
             if not status_format or status_format == "text":

scalyr_agent/builtin_monitors/docker_monitor.py

@@ -1483,6 +1483,7 @@ def __create_docker_logger(self, log, last_request):
             name,
             stream,
             log["log_config"]["path"],
+            log["log_config"],
             self._config,
             last_request,
         )
@@ -1504,6 +1505,7 @@ def __init__(
         name,
         stream,
         log_path,
+        log_config,
         config,
         last_request=None,
         max_log_size=20 * 1024 * 1024,
@@ -1516,6 +1518,7 @@ def __init__(
         # stderr or stdout
         self.stream = stream
         self.log_path = log_path
+        self.log_config = log_config
         self.stream_name = name + "-" + stream
 
         self.__max_previous_lines = config.get("max_previous_lines")