Merge commit '9dea4f9c2e9a4b7772adc3c9e5be079c66a18cfe' into release

echee2 · echee2 · commit 5518b0adb629 · 2019-09-19T17:32:26.000-07:00
diff --git a/.circleci/docker_unified_smoke_unit/Dockerfile.sanity.centos b/.circleci/docker_unified_smoke_unit/Dockerfile.sanity.centos
@@ -13,6 +13,7 @@ FROM centos
 RUN yum install -y epel-release && yum install -y python36 python36-pip
 RUN pip3 install requests
 RUN yum install -y perl
+RUN yum install -y sudo
 
 #------------------------------------------------------
 # Copy and run test scripts
diff --git a/.circleci/docker_unified_smoke_unit/Dockerfile.sanity.ubuntu b/.circleci/docker_unified_smoke_unit/Dockerfile.sanity.ubuntu
@@ -14,6 +14,7 @@ RUN apt-get update && apt-get install -y curl python python3
 RUN apt-get install -y python3-pip
 RUN apt-get install perl
 RUN pip3 install requests
+RUN apt-get install sudo
 
 #------------------------------------------------------
 # Copy and run test scripts
diff --git a/.circleci/smoketest_release_standalone.sh b/.circleci/smoketest_release_standalone.sh
@@ -60,8 +60,8 @@ elif [[ $ARTIFACT_FILE =~ .*deb ]]; then
   apt-get update
 elif [[ $ARTIFACT_FILE == "PUBLISHED" ]]; then
   echo "Installing via published script (https://www.scalyr.com/install-agent.sh)"
-  pushd /tmp && curl https://www.scalyr.com/install-agent.sh -o /tmp/install_agent.sh && chmod 755 /tmp/install_agent.sh && \
-   /tmp/install-agent.sh --set-api-key $SCALYR_API_KEY
+  pushd /tmp && curl https://www.scalyr.com/install-agent.sh -o /tmp/install-agent.sh && sleep 3 && chmod 755 /tmp/install-agent.sh && \
+  sudo /tmp/install-agent.sh --set-api-key $SCALYR_API_KEY
 elif [[ $ARTIFACT_FILE =~ .*gz ]]; then
   pushd /usr/share
   tar --no-same-owner -zxf $ARTIFACT_FILE
@@ -76,6 +76,8 @@ fi
 FILES=/tmp
 
 if [[ ! -f $SCALYR_AGENT_ETC_DIR/agent.json ]]; then
+    find / -name scalyr_install.log
+    cat $(find / -name scalyr_install.log)
     exit 1
 fi
 
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,6 +1,20 @@
 Scalyr Agent 2 Changes By Release
 =================================
 
+## 2.0.53 "Firefly" - Sep 18, 2019
+
+<!---
+Packaged by Imron Alston <imron@scalyr.com> on Sep 18, 2019 15:00 -0700
+--->
+
+Features
+* Collection of per CPU metric data under docker/kubernetes has been placed behind a config option to prevent collecting excessive metrics on machines with a large number of cores.
+
+Miscellaneous changes
+* Added scripts for bringing up/tearing down large clusters for testing
+* Fixed problem with smoketest scripts
+
+
 ## 2.0.52 "Einstein-Rosen Bridge" - Sep 16, 2019
 
 <!---
diff --git a/VERSION b/VERSION
@@ -1 +1 @@
-2.0.52
+2.0.53
diff --git a/k8s/scalyr-agent-2-envfrom.yaml b/k8s/scalyr-agent-2-envfrom.yaml
@@ -11,7 +11,7 @@ spec:
       serviceAccountName: scalyr-service-account
       containers:
       - name: scalyr-agent
-        image: scalyr/scalyr-k8s-agent:2.0.52
+        image: scalyr/scalyr-k8s-agent:2.0.53
         imagePullPolicy: Always
         env:
           - name: SCALYR_API_KEY
diff --git a/k8s/scalyr-agent-2.yaml b/k8s/scalyr-agent-2.yaml
@@ -11,7 +11,7 @@ spec:
       serviceAccountName: scalyr-service-account
       containers:
       - name: scalyr-agent
-        image: scalyr/scalyr-k8s-agent:2.0.52
+        image: scalyr/scalyr-k8s-agent:2.0.53
         imagePullPolicy: Always
         env:
           - name: SCALYR_API_KEY
diff --git a/performance/kops_cluster_setup/README.md b/performance/kops_cluster_setup/README.md
@@ -0,0 +1 @@
+Please see `sample_commands.txt`.
diff --git a/performance/kops_cluster_setup/agentscale.yaml b/performance/kops_cluster_setup/agentscale.yaml
@@ -26,7 +26,7 @@ apiVersion: kops/v1alpha2
 kind: Cluster
 metadata:
   creationTimestamp: null
-  name: agentscale.dev.scalyr.com
+  name: <YOUR_CUSTER_NAME>
 spec:
   api:
     loadBalancer:
@@ -35,7 +35,7 @@ spec:
     rbac: {}
   channel: stable
   cloudProvider: aws
-  configBase: s3://agentscale-dev-scalyr-com-state-store/agentscale.dev.scalyr.com
+  configBase: s3://<YOUR_BUCKET_STORE_NAME>/<YOUR_CUSTER_NAME>
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
@@ -78,7 +78,7 @@ spec:
   kubernetesApiAccess:
   - 0.0.0.0/0
   kubernetesVersion: 1.10.13
-  masterPublicName: api.agentscale.dev.scalyr.com
+  masterPublicName: api.<YOUR_CUSTER_NAME>
   networkCIDR: 172.20.0.0/16
   networking:
     kopeio: {}
@@ -128,7 +128,7 @@ spec:
     zone: us-east-1f
   topology:
     bastion:
-      bastionPublicName: bastion.agentscale.dev.scalyr.com
+      bastionPublicName: bastion.<YOUR_CUSTER_NAME>
       idleTimeoutSeconds: 3600
     dns:
       type: Public
@@ -142,7 +142,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1a
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
@@ -162,7 +162,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1b
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
@@ -182,7 +182,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1c
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
@@ -202,7 +202,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1d
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
@@ -222,7 +222,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1f
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
@@ -242,7 +242,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: nodes
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
@@ -266,7 +266,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: bastions
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-05-13
diff --git a/performance/kops_cluster_setup/create_agentscale_yaml.sh b/performance/kops_cluster_setup/create_agentscale_yaml.sh
@@ -6,8 +6,8 @@
 # Define variables
 export AWS_ACCESS_KEY_ID=$(aws configure get aws_access_key_id)
 export AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key)
-BUCKET=agentscale-dev-scalyr-com-state-store
-export KOPS_CLUSTER_NAME=agentscale.dev.scalyr.com
+BUCKET=<YOUR_BUCKET_STORE_NAME>
+export KOPS_CLUSTER_NAME=<YOUR_CLUSTER_NAME>  # should be a valid DNS name, e.g. agentscale.dev.scalyr.com
 export KOPS_STATE_STORE=s3://${BUCKET}
 
 # Create bucket for cluster state
diff --git a/performance/kops_cluster_setup/fivezones.yaml b/performance/kops_cluster_setup/fivezones.yaml
@@ -2,7 +2,7 @@ apiVersion: kops/v1alpha2
 kind: Cluster
 metadata:
   creationTimestamp: null
-  name: agentscale.dev.scalyr.com
+  name: <YOUR_CUSTER_NAME>
 spec:
   api:
     loadBalancer:
@@ -11,7 +11,7 @@ spec:
     rbac: {}
   channel: stable
   cloudProvider: aws
-  configBase: s3://agentscale-dev-scalyr-com-state-store/agentscale.dev.scalyr.com
+  configBase: s3://<YOUR_BUCKET_STORE_NAME>/<YOUR_CUSTER_NAME>
   etcdClusters:
   - cpuRequest: 200m
     etcdMembers:
@@ -49,7 +49,7 @@ spec:
   kubernetesApiAccess:
   - 0.0.0.0/0
   kubernetesVersion: 1.12.10
-  masterPublicName: api.agentscale.dev.scalyr.com
+  masterPublicName: api.<YOUR_CUSTER_NAME>
   networkCIDR: 172.20.0.0/16
   networking:
     kopeio: {}
@@ -99,7 +99,7 @@ spec:
     zone: us-east-1f
   topology:
     bastion:
-      bastionPublicName: bastion.agentscale.dev.scalyr.com
+      bastionPublicName: bastion.<YOUR_CUSTER_NAME>
     dns:
       type: Public
     masters: private
@@ -112,7 +112,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1a
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-06-21
@@ -132,7 +132,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1b
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-06-21
@@ -152,7 +152,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1c
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-06-21
@@ -172,7 +172,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1d
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-06-21
@@ -192,7 +192,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: master-us-east-1f
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-06-21
@@ -212,7 +212,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: nodes
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-06-21
@@ -236,7 +236,7 @@ kind: InstanceGroup
 metadata:
   creationTimestamp: null
   labels:
-    kops.k8s.io/cluster: agentscale.dev.scalyr.com
+    kops.k8s.io/cluster: <YOUR_CUSTER_NAME>
   name: bastions
 spec:
   image: kope.io/k8s-1.12-debian-stretch-amd64-hvm-ebs-2019-06-21
diff --git a/performance/kops_cluster_setup/sample_commands.txt b/performance/kops_cluster_setup/sample_commands.txt
@@ -2,12 +2,24 @@
 # Author: echee@scalyr.com (Edward Chee)
 #
 # Prerequisites:
+#
 # 1) Install kops
 # 2) Create and version an s3 bucket.  Create the agentscale.yaml cluster definition file 
 #    Run `create_agentscale_yaml.sh` to do this.  The cluster definition will be saved to `fivezones.yaml`.
 # 3) Save `fivezones.yaml` as `agentscale.yaml`.  Customize `agentscale.yaml`, e.g turning off Gossip protocol.
 #    (Diff `fivezones.yaml` and `agentscale.yaml` to view the customizations I used to 
 #    successfully bring up a 1000-node cluster.)
+#
+# Finally, note that all scripts and sample files have the following placeholders:
+#  <YOUR_CUSTER_NAME> : must be a valid DNS name, e.g. agentscale.dev.scalyr.com (pick something unique to your org)
+#  <YOUR_BUCKET_STORE_NAME> : s3 bucket name
+#
+# These placeholders were retroactively edited in the generated `fivezones.yaml` and `agentscale.yaml` to prevent
+# inadvertent reuse of the same s3 bucket by multiple parties.
+#
+# If you wish to use the `agentscale.yaml` directly, you must replace those placeholders with your custom values.
+# Otherwise, set them and run the sample commands in this script to generate `fivezones.yaml`, then `agentscale.yaml`.
+#
 ###################################################################################################################
 
 ###################################################################################################################
@@ -21,8 +33,8 @@ export AWS_SECRET_ACCESS_KEY=$(aws configure get aws_secret_access_key)
 ###################################################################################################################
 # kops environment variables. These must match the settings you used in step 2 to create the s3 bucket
 ###################################################################################################################
-BUCKET=agentscale-dev-scalyr-com-state-store
-export KOPS_CLUSTER_NAME=agentscale.dev.scalyr.com
+BUCKET=<YOUR_BUCKET_STORE_NAME>
+export KOPS_CLUSTER_NAME=<YOUR_CUSTER_NAME>
 export KOPS_STATE_STORE=s3://${BUCKET}
 
 ###################################################################################################################
@@ -33,12 +45,12 @@ kops create -f agentscale.yaml
 ###################################################################################################################
 # Grant your ssh public key access to the cluster
 ###################################################################################################################
-kops create secret --name agentscale.dev.scalyr.com sshpublickey admin -i ~/.ssh/agentscale_id_rsa.pub
+kops create secret --name $KOPS_CLUSTER_NAME sshpublickey admin -i ~/.ssh/agentscale_id_rsa.pub
 
 ###################################################################################################################
 # Create the cluster
 ###################################################################################################################
-kops update cluster --name agentscale.dev.scalyr.com --yes
+kops update cluster --name $KOPS_CLUSTER_NAME --yes
 
 ###################################################################################################################
 # Wait a while then check cluster status
@@ -49,4 +61,4 @@ kubectl get nodes
 ###################################################################################################################
 # When done, delete the cluster
 ###################################################################################################################
-kops delete cluster --name agentscale.dev.scalyr.com --yes
+kops delete cluster --name $KOPS_CLUSTER_NAME --yes
diff --git a/scalyr_agent/builtin_monitors/docker_monitor.py b/scalyr_agent/builtin_monitors/docker_monitor.py
@@ -77,6 +77,11 @@
                      'Optional (defaults to docker). Prefix added to the start of all docker logs. ',
                      convert_to=str, default='docker', env_aware=True)
 
+define_config_option( __monitor__, 'docker_percpu_metrics',
+                     'Optional (defaults to False). When `True`, emits cpu usage stats per core.  Note: This is disabled by '
+                     'default because it can result in an excessive amount of metric data on cpus with a large number of cores.',
+                     convert_to=bool, default=False, env_aware=True)
+
 define_config_option( __monitor__, 'max_previous_lines',
                      'Optional (defaults to 5000). The maximum number of lines to read backwards from the end of the stdout/stderr logs\n'
                      'when starting to log a containers stdout/stderr to find the last line that was sent to Scalyr.',
@@ -1479,6 +1484,8 @@ def _initialize( self ):
 
         self.__report_container_metrics = self._config.get('report_container_metrics')
 
+        self.__percpu_metrics = self._config.get('docker_percpu_metrics')
+
         metrics_only = self._config.get('metrics_only')
 
         self.label_exclude_globs = self._config.get( 'label_exclude_globs' )
@@ -1607,13 +1614,13 @@ def __log_memory_stats_metrics( self, container, metrics ):
     def __log_cpu_stats_metrics( self, container, metrics ):
         if 'cpu_usage' in metrics:
             cpu_usage = metrics['cpu_usage']
-            if 'percpu_usage' in cpu_usage:
+            if self.__percpu_metrics and 'percpu_usage' in cpu_usage:
                 percpu = cpu_usage['percpu_usage']
                 count = 1
                 if percpu:
                     for usage in percpu:
                         extra = { 'cpu' : count }
-                        self._logger.emit_value( 'docker.cpu.usage', usage, monitor_id_override=container )
+                        self._logger.emit_value( 'docker.cpu.usage', usage, extra, monitor_id_override=container )
                         count += 1
             self.__log_metrics( container, self.__cpu_usage_metrics, cpu_usage )
 
diff --git a/scalyr_agent/builtin_monitors/kubernetes_monitor.py b/scalyr_agent/builtin_monitors/kubernetes_monitor.py
diff --git a/scalyr_agent/tests/configuration_docker_test.py b/scalyr_agent/tests/configuration_docker_test.py
diff --git a/scalyr_agent/tests/configuration_k8s_test.py b/scalyr_agent/tests/configuration_k8s_test.py
