Add opt-in default metrics to CloudWatch agent

Report on memory and disk usage

Author: ouranos

packer/linux/conf/bin/bk-configure-cloudwatch-agent.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+# shellcheck disable=SC2094
+set -euo pipefail
+
+if [[ "${CLOUDWATCH_ENABLE_METRICS:-false}" == "true" ]]; then
+    cw_config="/opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json"
+    cat <<<"$(jq \
+        --arg queue "$BUILDKITE_QUEUE" \
+        '. + {
+            metrics: {
+                metrics_collected: {
+                    mem: {measurement: ["mem_used_percent"], append_dimensions: {BuildkiteQueue: $queue}},
+                    disk: {measurement: ["used_percent"], resources: ["*"], append_dimensions: {BuildkiteQueue: $queue}}
+                },
+                append_dimensions: {
+                    AutoScalingGroupName: "${aws:AutoScalingGroupName}"
+                }
+            }
+        }' $cw_config)" >$cw_config
+fi
+
+# Enable and start amazon-cloudwatch-agent
+systemctl enable amazon-cloudwatch-agent
+systemctl start amazon-cloudwatch-agent

packer/linux/scripts/install-cloudwatch-agent.sh

@@ -2,16 +2,12 @@
 
 set -eu -o pipefail
 
-
 echo "Installing cloudwatch agent..."
 sudo yum install -y amazon-cloudwatch-agent
 
 echo "Adding amazon-cloudwatch-agent config..."
 sudo cp /tmp/conf/cloudwatch-agent/config.json /opt/aws/amazon-cloudwatch-agent/etc/amazon-cloudwatch-agent.json
 
-echo "Configuring amazon-cloudwatch-agent to start at boot"
-sudo systemctl enable amazon-cloudwatch-agent
-
 # These will send some systemctl service logs (like the buildkite agent and docker) to logfiles
 echo "Adding rsyslogd configs..."
 sudo cp /tmp/conf/cloudwatch-agent/rsyslog.d/* /etc/rsyslog.d/

templates/aws-stack.yml

@@ -81,6 +81,7 @@ Metadata:
         - InstanceRolePermissionsBoundaryARN
         - IMDSv2Tokens
         - EnableDetailedMonitoring
+        - EnableCloudWatchMetrics
 
       - Label:
           default: Auto-scaling Configuration
@@ -437,6 +438,14 @@ Parameters:
       - "false"
     Default: "false"
 
+  EnableCloudWatchMetrics:
+    Type: String
+    Description: Enable CloudWatch agent metrics
+    AllowedValues:
+      - "true"
+      - "false"
+    Default: "false"
+
   EnableCostAllocationTags:
     Type: String
     Description: Enables AWS Cost Allocation tags for all resources in the stack. See https://docs.aws.amazon.com/awsaccountbilling/latest/aboutv2/cost-alloc-tags.html
@@ -1050,6 +1059,11 @@ Resources:
                   BUILDKITE_ENABLE_INSTANCE_STORAGE="${EnableInstanceStorage}" \
                     /usr/local/bin/bk-configure-docker.sh
                   --==BOUNDARY==
+                  Content-Type: text/cloud-boothook; charset="us-ascii"
+                  CLOUDWATCH_ENABLE_METRICS="${EnableCloudWatchMetrics}" \
+                  BUILDKITE_QUEUE="${BuildkiteQueue}" \
+                    /usr/local/bin/bk-configure-cloudwatch-agent.sh
+                  --==BOUNDARY==
                   Content-Type: text/x-shellscript; charset="us-ascii"
                   #!/bin/bash -v
                   BUILDKITE_STACK_NAME="${AWS::StackName}" \