log-collector

#!/bin/bash

# This script leverages the AWS CLI to collect Elastic Stack logs for your
# Buildkite Elastic CI Stack instances.
#
# It assumes you have the AWS CLI tool configured, with appropriate credentials
# so it is recommended to use something like aws-vault: https://github.com/99designs/aws-vault.
#
# You will need to know the Instance ID and Stack name of the resources
# you want to collect logs for.
#
# Once you have collected the logs, please send them to support@buildkite.com
# with a description of the issue you are seeing, and we'll be happy to assist!

set -euo pipefail

stackName=""
instanceId=""
outputFormat="text"
region=""
days=1

now=""
dir=""
startTime=""
errFile=""

usage() {
  cat <<EOF
Usage: $0 -s <stack-name> [-i <instance-id>] [-o <output-format>] [-r <region>] [-d <days>]
Collect Buildkite Elastic CI Stack logs from AWS CloudWatch.

  -s   Stack name (required).
  -i   Instance ID (optional). When set, also collects that instance's CloudWatch logs.
  -o   Output format: text (default), json, or table.
  -r   AWS region (optional). Defaults to your AWS CLI/IAM Role config.
  -d   Days of logs to collect (1-365). Defaults to 1 day.
  -h   Show help.
EOF
}

parse_args() {
  while getopts ':hs:i:o:r:d:' opt; do
    case ${opt} in
    s) stackName=$OPTARG ;;
    i) instanceId=$OPTARG ;;
    o) outputFormat=$OPTARG ;;
    r) region=$OPTARG ;;
    d) days=$OPTARG ;;
    h)
      usage
      exit 0
      ;;
    \?)
      echo "Invalid option: -$OPTARG" 1>&2
      exit 1
      ;;
    :)
      echo "Invalid option: -$OPTARG requires an argument" 1>&2
      exit 1
      ;;
    esac
  done

  if [[ -z "$stackName" ]]; then
    echo "Error: -s <stack-name> is required" 1>&2
    usage
    exit 1
  fi

  if [[ -n "$instanceId" && ! "$instanceId" =~ ^i-[0-9a-f]{8}([0-9a-f]{9})?$ ]]; then
    echo "Error: -i must be an EC2 instance ID (got '$instanceId')" 1>&2
    exit 1
  fi

  if ! [[ "$days" =~ ^[1-9][0-9]{0,3}$ ]] || ((days > 365)); then
    echo "Error: -d must be a whole number of days from 1 to 365 (got '$days')" 1>&2
    exit 1
  fi

  case "$outputFormat" in
  text | json | table) ;;
  *)
    echo "Error: -o must be one of text, json, table (got '$outputFormat')" 1>&2
    exit 1
    ;;
  esac
}

aws_cli() {
  if [[ -n "$region" ]]; then
    aws --region "$region" "$@"
  else
    aws "$@"
  fi
}

record_error() {
  echo "ERROR: $1" 1>&2
  echo "$1" >>"$dir/COLLECTION-ERRORS.txt"
}

mark_empty() {
  echo "$2" >"$dir/$1.EMPTY"
  rm -f "$dir/$1"
}

mark_failed() {
  echo "$2" >"$dir/$1.ERROR"
  rm -f "$dir/$1"
}

is_empty() {
  [[ ! -s "$1" ]] && return 0
  [[ "$(wc -c <"$1" | tr -d ' ')" -le 8 && "$(tr -d '[:space:]' <"$1")" == "[]" ]]
}

last_error() {
  tr -s '[:space:]' ' ' <"$errFile" 2>/dev/null | sed 's/^ //; s/ $//' || true
}

missing_log_group() {
  [[ "$1" == *ResourceNotFoundException* ]]
}

compute_start_time() {
  if date --version >/dev/null 2>&1; then
    date --date="${days} days ago" "+%s000"
  else
    date -v-"${days}"d "+%s000"
  fi
}

collect_instance() {
  echo "Collecting logs for instance $instanceId..."
  local name safe file err
  for name in buildkite-agent system lifecycled docker-daemon cloud-init cloud-init/output cfn-init elastic-stack auth EC2Launch/UserdataExecution; do
    safe="${name//\//-}"
    file="$dir/$safe-$instanceId.log"
    if ! aws_cli logs filter-log-events \
      --log-group-name "/buildkite/$name" \
      --log-stream-names "$instanceId" \
      --start-time "$startTime" \
      --query "events[].[timestamp,message]" \
      --output "$outputFormat" >"$file" 2>"$errFile"; then
      err=$(last_error)
      if missing_log_group "$err"; then
        mark_empty "$safe-$instanceId.log" "Log group /buildkite/$name does not exist for $instanceId"
      else
        record_error "Failed to read /buildkite/$name for $instanceId: $err"
        mark_failed "$safe-$instanceId.log" "Collection failed for /buildkite/$name: $err"
      fi
      continue
    fi
    if is_empty "$file"; then
      mark_empty "$safe-$instanceId.log" "No log events in /buildkite/$name for $instanceId within the last $days day(s)"
    fi
  done
}

list_lambdas() {
  local stack="$1" out
  if out=$(aws_cli cloudformation list-stack-resources \
    --stack-name "$stack" \
    --query "StackResourceSummaries[?ResourceType=='AWS::Lambda::Function'].PhysicalResourceId" \
    --output text 2>"$errFile"); then
    echo "$out"
  else
    record_error "Failed to list Lambda functions in $stack: $(last_error)"
  fi
}

find_lambdas() {
  local stack="$1" nested child
  list_lambdas "$stack"

  if ! nested=$(aws_cli cloudformation list-stack-resources \
    --stack-name "$stack" \
    --query "StackResourceSummaries[?ResourceType=='AWS::CloudFormation::Stack' || ResourceType=='AWS::Serverless::Application'].PhysicalResourceId" \
    --output text 2>"$errFile" | tr '\t' '\n'); then
    record_error "Failed to list nested stacks in $stack: $(last_error)"
    return 0
  fi

  while IFS= read -r child; do
    [[ -z "$child" ]] && continue
    find_lambdas "$child"
  done <<<"$nested"
}

find_log_group() {
  local fn="$1" lg
  if ! lg=$(aws_cli lambda get-function-configuration \
    --function-name "$fn" \
    --query "LoggingConfig.LogGroup" \
    --output text 2>"$errFile"); then
    record_error "Failed to resolve log group for $fn at /aws/lambda/$fn: $(last_error)"
    lg=""
  fi
  if [[ -z "$lg" || "$lg" == "None" ]]; then
    lg="/aws/lambda/$fn"
  fi
  echo "$lg"
}

collect_lambda() {
  local fn="$1" safe logGroup file err
  safe="${fn//\//-}"
  logGroup=$(find_log_group "$fn")
  file="${dir}/lambda-${safe}.log"
  echo "Collecting logs for Lambda $fn..."
  if ! aws_cli logs filter-log-events \
    --log-group-name "$logGroup" \
    --start-time "$startTime" \
    --query "events[].[timestamp,message]" \
    --output "$outputFormat" >"$file" 2>"$errFile"; then
    err=$(last_error)
    if missing_log_group "$err"; then
      mark_empty "lambda-${safe}.log" "Log group $logGroup does not exist for $fn"
    else
      record_error "Failed to read $logGroup for $fn: $err"
      mark_failed "lambda-${safe}.log" "Collection failed for $logGroup: $err"
    fi
    return 0
  fi
  if is_empty "$file"; then
    mark_empty "lambda-${safe}.log" "No log events in $logGroup within the last $days day(s)"
  fi
}

collect_lambdas() {
  local functions fn
  functions=$(find_lambdas "$stackName" | tr '\t' '\n' | sort -u) || true

  if [[ -z "$functions" ]]; then
    record_error "No Lambda functions found in stack $stackName"
    mark_empty "lambda.log" "No Lambda functions found in stack $stackName"
    return 0
  fi

  while IFS= read -r fn; do
    [[ -z "$fn" ]] && continue
    collect_lambda "$fn"
  done <<<"$functions"
}

collect_asg() {
  local asg file err
  if ! asg=$(aws_cli cloudformation describe-stack-resources \
    --stack-name "$stackName" \
    --logical-resource-id AgentAutoScaleGroup \
    --query "StackResources[*].PhysicalResourceId" \
    --output text 2>"$errFile") || [[ -z "$asg" ]]; then
    err=$(last_error)
    record_error "Failed to resolve AgentAutoScaleGroup in $stackName: $err"
    mark_failed "asg-scaling.log" "Collection failed resolving AgentAutoScaleGroup in $stackName: $err"
    return 0
  fi

  echo "Collecting scaling activity for $asg..."
  file="${dir}/asg-scaling.log"
  if ! aws_cli autoscaling describe-scaling-activities \
    --auto-scaling-group-name "$asg" \
    --query "Activities" \
    --output "$outputFormat" >"$file" 2>"$errFile"; then
    err=$(last_error)
    record_error "Failed to collect scaling activity for $asg: $err"
    mark_failed "asg-scaling.log" "Collection failed for $asg: $err"
    return 0
  fi
  if is_empty "$file"; then
    mark_empty "asg-scaling.log" "No scaling activity found for $asg"
  fi
}

package() {
  local archive
  archive="$(basename "$dir").tar.gz"
  tar -czf "$archive" "$dir"
  echo "Finished collecting logs, you can find them at $PWD/$archive"
}

summarise() {
  local errors=0 logs
  if [[ -f "$dir/COLLECTION-ERRORS.txt" ]]; then
    errors=$(wc -l <"$dir/COLLECTION-ERRORS.txt" | tr -d ' ')
  fi
  logs=$(find "$dir" -maxdepth 1 -type f -name '*.log' | wc -l | tr -d ' ')

  if ((errors > 0)); then
    echo "WARNING: collection completed with $errors error(s). See COLLECTION-ERRORS.txt for more details." 1>&2
  fi
  if ((logs == 0 && errors > 0)); then
    echo "ERROR: every source failed." 1>&2
    exit 1
  fi
}

main() {
  parse_args "$@"

  now=$(date +"%F-%H%M%S")
  dir=$(mktemp -d buildkite-logs-"$now"-XXXX)
  trap 'rm -rf "$dir" "$errFile"' EXIT
  errFile=$(mktemp)
  startTime=$(compute_start_time)

  local stackError
  if ! stackError=$(aws_cli cloudformation describe-stacks --stack-name "$stackName" 2>&1 >/dev/null); then
    echo "Error: could not describe CloudFormation stack '$stackName'" 1>&2
    echo "$stackError" 1>&2
    exit 1
  fi

  if [[ -n "$instanceId" ]]; then
    collect_instance
  else
    echo "No instance ID supplied, skipping per-instance logs"
  fi

  collect_lambdas
  collect_asg

  package
  summarise
}

main "$@"