Enable ADOT and OpenTelemetry tracing in forms-runner

Configure the ADOT sidecar for forms-runner and forms-queue-worker, and
enable OpenTelemetry in the application containers.

This is only enabled in dev and staging for now.

This specifically configures OpenTelemetry to export traces to AWS X-Ray.

A new buildspec was required for db-migrate, as the original task failed
when the ADOT sidecar was added to the main forms-runner task definition.
The new version retrieves the app task from the existing task definition,
rather than just fetching the first container definition.

This also makes it possible to enable the ADOT sidecar and OpenTelemetry
in other apps. Only forms-runner is configured to use it for now.

Author: whi-tw

infra/deployments/forms/forms-runner/main.tf

@@ -49,6 +49,7 @@ module "forms_runner" {
   enable_maintenance_mode                     = var.forms_runner_settings.enable_maintenance_mode
   cloudwatch_metrics_enabled                  = var.forms_runner_settings.cloudwatch_metrics_enabled
   analytics_enabled                           = var.forms_runner_settings.analytics_enabled
+  enable_opentelemetry                        = var.forms_runner_settings.enable_opentelemetry
   deploy_account_id                           = var.deploy_account_id
   ses_submission_email_from_email_address     = var.forms_runner_settings.ses_submission_email_from_email_address
   ses_submission_email_reply_to_email_address = var.forms_runner_settings.ses_submission_email_reply_to_email_address

infra/deployments/forms/inputs.tf

@@ -166,6 +166,7 @@ variable "forms_runner_settings" {
     enable_maintenance_mode                                         = bool
     cloudwatch_metrics_enabled                                      = bool
     analytics_enabled                                               = bool
+    enable_opentelemetry                                            = optional(bool, false)
     allow_human_readonly_roles_to_assume_submissions_to_s3_role     = bool
     allow_human_readonly_roles_to_assume_submissions_to_runner_role = bool
     ses_submission_email_from_email_address                         = string

infra/deployments/forms/pipelines/buildspecs/db-migrate/db-migrate-adot.yml

@@ -0,0 +1,75 @@
+version: 0.2
+phases:
+  pre_build:
+    commands:
+      # Check Image URI is not the default pipeline value
+      - |
+        if [[ "${IMAGE_URI}" = "MUST_BE_SET" ]]; then
+          echo "The IMAGE_URI has not been set by the caller. The value of IMAGE_URI is \"${IMAGE_URI}\""
+          exit 1
+        fi
+
+      # Get task definition
+      - echo "Existing ECS task definition name is ${TASK_DEFINITION_NAME}"
+      - ECS_TASK_DEFINITION=$(aws ecs describe-task-definition --task-definition "${TASK_DEFINITION_NAME}")
+
+      # Delete any reference to the old image from the task definition and add the new image uri
+      - NEW_ECS_TASK_DEFINITION=$(echo "${ECS_TASK_DEFINITION}" | jq --arg "INPUT_IMAGE_URI" "${IMAGE_URI}" --arg "APP_NAME" "${APP_NAME}" '.taskDefinition | .containerDefinitions |= map(if .name == $APP_NAME then .image = $INPUT_IMAGE_URI else . end) | del(.taskDefinitionArn) | del(.revision) | del(.status) | del(.requiresAttributes) | del(.compatibilities) |  del(.registeredAt)  | del(.registeredBy)')
+      - echo "Replaced image uri with ${IMAGE_URI} for container ${APP_NAME}"
+
+      # Register a new task definition
+      - |
+        NEW_ECS_TASK_DEFINITION_ARN=$(aws ecs register-task-definition --cli-input-json "$NEW_ECS_TASK_DEFINITION" | jq -r '.taskDefinition.taskDefinitionArn' )
+        echo "New ECS task definition ARN ${NEW_ECS_TASK_DEFINITION_ARN}"
+
+      # Get the existing configuration
+      - ECS_CLUSTER_ARN=$(aws ecs describe-clusters --cluster "${CLUSTER_NAME}" | jq -r '.clusters[].clusterArn')
+      - ECS_TASK_CONTAINER_DEFINITION_NAME=$(aws ecs describe-task-definition --task-definition "${TASK_DEFINITION_NAME}" | jq --arg "APP_NAME" "${APP_NAME}" -r '.taskDefinition.containerDefinitions[] | select(.name == $APP_NAME) | .name')
+      - ECS_TASK_NETWORK_CONFIGURATION=$(aws ecs describe-services --cluster "${CLUSTER_NAME}" --services "${APP_NAME}" | jq '.services[].networkConfiguration[]')
+
+      # Write new task definition overriding the command the container runs
+      - |
+        jq -nrM \
+        --arg "ECS_CLUSTER_ARN" "${ECS_CLUSTER_ARN}" \
+        --arg "CONTAINER_DEFINITION_NAME" "${ECS_TASK_CONTAINER_DEFINITION_NAME}" \
+        --arg "ECS_TASK_DEFINITION_ARN" "${NEW_ECS_TASK_DEFINITION_ARN}" \
+        --argjson "ECS_TASK_NETWORK_CONFIGURATION" "${ECS_TASK_NETWORK_CONFIGURATION}" \
+        '{
+          "cluster": $ECS_CLUSTER_ARN,
+          "taskDefinition": $ECS_TASK_DEFINITION_ARN,
+          "count": 1,
+          "launchType": "FARGATE",
+          "overrides": {
+              "containerOverrides": [
+                  {
+                      "name": $CONTAINER_DEFINITION_NAME,
+                      "command": ["rake", "db:migrate"],
+                      "environment": [
+                          { "name": "VERBOSE", "value": "true" }
+                      ]
+                  }
+              ]
+          },
+          "networkConfiguration": {
+              "awsvpcConfiguration": $ECS_TASK_NETWORK_CONFIGURATION
+          }
+        }' > db_migrate_task_definition.json
+
+      - echo -e "New task definition \n $(<db_migrate_task_definition.json )"
+  build:
+    commands:
+      - echo "Running database migration for ${APP_NAME}"
+      - RUNNING_TASK_ARN=$(aws ecs run-task --cli-input-json "file://db_migrate_task_definition.json" | jq -r '.tasks[].taskArn')
+      - echo "Running task ARN ${RUNNING_TASK_ARN}"
+      - echo "Waiting for the task to finish"
+      - aws ecs wait tasks-stopped --tasks "${RUNNING_TASK_ARN}" --cluster "${ECS_CLUSTER_ARN}"
+      # Determine the exit code for the running task
+      # Determine if the running task or the CodeBuild build failed
+      # No failures: 0
+      # Any failures: 1
+      - |
+        RUNNING_TASK_EXIT_CODE=$(\
+          aws ecs describe-tasks --tasks "${RUNNING_TASK_ARN}" --cluster "${ECS_CLUSTER_ARN}" \
+          | jq --arg "CONTAINER_NAME" "${ECS_TASK_CONTAINER_DEFINITION_NAME}" -r '.tasks[0].containers[] | select(.name == $CONTAINER_NAME) | .exitCode'
+          )
+      - exit "${RUNNING_TASK_EXIT_CODE}"

infra/deployments/forms/pipelines/deploy-forms-runner-container.tf

@@ -328,7 +328,7 @@ module "db_migrate_runner" {
   project_description        = "Run database migrations"
   environment                = var.environment_name
   artifact_store_arn         = module.artifact_bucket.arn
-  buildspec                  = file("${path.root}/buildspecs/db-migrate/db-migrate.yml")
+  buildspec                  = file("${path.root}/buildspecs/db-migrate/db-migrate-adot.yml")
   log_group_name             = "codebuild/db_migrate_runner_${var.environment_name}"
   codebuild_service_role_arn = data.aws_iam_role.deployer_role.arn
 }

infra/deployments/forms/tfvars/dev.tfvars

@@ -96,6 +96,7 @@ forms_runner_settings = {
   enable_maintenance_mode                                         = false
   cloudwatch_metrics_enabled                                      = true
   analytics_enabled                                               = true
+  enable_opentelemetry                                            = true
   allow_human_readonly_roles_to_assume_submissions_to_s3_role     = true
   allow_human_readonly_roles_to_assume_submissions_to_runner_role = true
   ses_submission_email_from_email_address                         = "no-reply@dev.forms.service.gov.uk"

infra/deployments/forms/tfvars/staging.tfvars

@@ -61,6 +61,7 @@ forms_runner_settings = {
   enable_maintenance_mode                                         = false
   cloudwatch_metrics_enabled                                      = true
   analytics_enabled                                               = true
+  enable_opentelemetry                                            = true
   allow_human_readonly_roles_to_assume_submissions_to_s3_role     = false
   allow_human_readonly_roles_to_assume_submissions_to_runner_role = false
   ses_submission_email_from_email_address                         = "no-reply@staging.forms.service.gov.uk"

infra/modules/deployer-access/manage-ecs-service-policy.tf

@@ -138,8 +138,11 @@ data "aws_iam_policy_document" "ecs" {
     ]
     resources = [
       "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-admin-ecs-task-policy",
+      "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-admin-adot-collector",
       "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-runner-ecs-task-policy",
-      "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-product-page-ecs-task-policy"
+      "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-runner-adot-collector",
+      "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-product-page-ecs-task-policy",
+      "arn:aws:iam::${var.account_id}:policy/${var.environment_name}-forms-product-page-adot-collector"
     ]
     effect = "Allow"
   }
@@ -281,9 +284,12 @@ data "aws_iam_policy_document" "logs" {
     ]
     resources = [
       "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-admin-${var.environment_name}:*",
+      "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-admin-${var.environment_name}/adot-collector:*",
       "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-runner-${var.environment_name}:*",
+      "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-runner-${var.environment_name}/adot-collector:*",
       "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-runner-queue-worker-${var.environment_name}:*",
       "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-product-page-${var.environment_name}:*",
+      "arn:aws:logs:eu-west-2:${var.account_id}:log-group:/aws/ecs/forms-product-page-${var.environment_name}/adot-collector:*",
     ]
     effect = "Allow"
   }

infra/modules/ecs-service/ecs.tf

@@ -11,18 +11,39 @@ data "aws_ecs_container_definition" "active_container" {
 
 locals {
   log_group_name         = "/aws/ecs/${var.application}-${var.env_name}"
+  adot_log_group_name    = "/aws/ecs/${var.application}-${var.env_name}/adot-collector"
   task_definition_family = "${var.env_name}_${var.application}"
 
   image = var.image == null ? data.aws_ecs_container_definition.active_container[0].image : var.image
 
   task_container_definition = {
-    name                   = var.application,
-    environment            = var.environment_variables,
+    name = var.application,
+    environment = var.enable_opentelemetry ? concat(var.environment_variables, [
+      {
+        name  = "ENABLE_OTEL"
+        value = "true"
+      },
+      {
+        name  = "OTEL_EXPORTER_OTLP_ENDPOINT"
+        value = "http://localhost:4318"
+      },
+      {
+        name  = "OTEL_SERVICE_NAME"
+        value = var.application
+      },
+      {
+        name  = "OTEL_PROPAGATORS"
+        value = "xray"
+      }
+    ]) : var.environment_variables,
     mountPoints            = [],
     secrets                = var.secrets,
     image                  = local.image
     essential              = true,
     readonlyRootFilesystem = var.readonly_root_filesystem
+    command                = null,
+    cpu                    = null,
+    memory                 = null,
     portMappings = [
       {
         hostPort      = var.container_port,
@@ -40,8 +61,54 @@ locals {
         awslogs-stream-prefix = local.log_group_name
       }
     },
+    healthCheck = null,
+    dependsOn = var.enable_opentelemetry ? [
+      {
+        containerName = "aws-otel-collector",
+        condition     = "START"
+      }
+    ] : []
   }
 
+  # ADOT collector sidecar container
+  adot_container_definition = {
+    name                   = "aws-otel-collector",
+    image                  = var.adot_image,
+    essential              = true,
+    readonlyRootFilesystem = false,
+    command = [
+      "--config=${var.adot_collector_config}"
+    ],
+    cpu    = var.adot_sidecar_cpu,
+    memory = var.adot_sidecar_memory,
+    logConfiguration = {
+      logDriver = "awslogs",
+      options = {
+        awslogs-group         = local.adot_log_group_name,
+        awslogs-region        = "eu-west-2",
+        awslogs-stream-prefix = "adot"
+      }
+    },
+    healthCheck = {
+      command = [
+        "CMD",
+        "/healthcheck"
+      ],
+      interval    = 5,
+      timeout     = 6,
+      retries     = 5,
+      startPeriod = 1
+    },
+  }
+
+  # Conditional container array composition
+  container_definitions = var.enable_opentelemetry ? jsonencode([
+    local.task_container_definition,
+    local.adot_container_definition
+    ]) : jsonencode([
+    local.task_container_definition
+  ])
+
   # Extract the values needed for the ECS service network configuration
   # to local variable so we can ensure the same configuration is used
   # for any pre-deploy tasks
@@ -53,7 +120,7 @@ locals {
 }
 resource "aws_ecs_task_definition" "task" {
   family                = local.task_definition_family
-  container_definitions = jsonencode([local.task_container_definition])
+  container_definitions = local.container_definitions
 
   // As this terraform module doesn't deal with updating app code, we see drift every time it's applied because the image is changed elsewhere.
   // Enable tracking of the latest ACTIVE task definition revision rather than the one in terraform state, so that changes to the image / task revision outside of terraform are picked up and not considered drift.
@@ -70,8 +137,11 @@ resource "aws_ecs_task_definition" "task" {
   task_role_arn      = aws_iam_role.ecs_task_role.arn
 
   requires_compatibilities = ["FARGATE"]
-  cpu                      = var.cpu
-  memory                   = var.memory
+  # When ADOT sidecar is enabled, round up to next valid Fargate CPU/memory configuration
+  # Valid Fargate configs: 256/.5-2GB, 512/1-4GB, 1024/2-8GB, 2048/4-16GB, 4096/8-30GB
+  # For forms-runner: 512 CPU + 1024 MB + ADOT (256 CPU + 512 MB) = needs 1024 CPU / 2048 MB minimum
+  cpu    = var.enable_opentelemetry ? (var.cpu + var.adot_sidecar_cpu <= 512 ? 512 : 1024) : var.cpu
+  memory = var.enable_opentelemetry ? (var.memory + var.adot_sidecar_memory <= 1024 ? 1024 : 2048) : var.memory
 
   network_mode = "awsvpc"
 

infra/modules/ecs-service/iam-adot.tf

@@ -0,0 +1,33 @@
+# IAM permissions for AWS Distro for OpenTelemetry sidecar
+# These permissions allow the ADOT collector to send traces to X-Ray
+# Note: Container logs are handled by the task execution role, not the task role
+
+data "aws_iam_policy_document" "adot_permissions" {
+  count = var.enable_opentelemetry ? 1 : 0
+
+  # X-Ray permissions for trace collection
+  statement {
+    sid    = "XRayAccess"
+    effect = "Allow"
+    actions = [
+      "xray:PutTraceSegments",
+      "xray:PutTelemetryRecords",
+      "xray:GetSamplingRules",
+      "xray:GetSamplingTargets",
+      "xray:GetSamplingStatisticSummaries"
+    ]
+    resources = ["*"]
+  }
+}
+
+resource "aws_iam_policy" "adot_policy" {
+  count  = var.enable_opentelemetry ? 1 : 0
+  name   = "${var.env_name}-${var.application}-adot-collector"
+  policy = data.aws_iam_policy_document.adot_permissions[0].json
+}
+
+resource "aws_iam_role_policy_attachment" "adot_policy_attachment" {
+  count      = var.enable_opentelemetry ? 1 : 0
+  role       = aws_iam_role.ecs_task_role.name
+  policy_arn = aws_iam_policy.adot_policy[0].arn
+}

infra/modules/ecs-service/logging.tf

@@ -5,6 +5,15 @@ resource "aws_cloudwatch_log_group" "log" {
   retention_in_days = 30
 }
 
+# Separate log group for ADOT collector
+resource "aws_cloudwatch_log_group" "adot_log" {
+  count = var.enable_opentelemetry ? 1 : 0
+  #checkov:skip=CKV_AWS_338:We're happy with 30 days retention for now
+  #checkov:skip=CKV_AWS_158:Default AWS SSE is sufficient, no need for CM KMS.
+  name              = local.adot_log_group_name
+  retention_in_days = 30
+}
+
 module "cribl_well_known" {
   source = "../well-known/cribl"
 }
@@ -21,3 +30,17 @@ resource "aws_cloudwatch_log_subscription_filter" "via_cribl_to_splunk" {
   distribution    = "ByLogStream"
   role_arn        = var.kinesis_subscription_role_arn
 }
+
+# Subscribe ADOT logs to Cribl/Splunk
+resource "aws_cloudwatch_log_subscription_filter" "adot_via_cribl_to_splunk" {
+  count = var.enable_opentelemetry && var.kinesis_subscription_role_arn != "" ? 1 : 0
+
+  name = "adot-via-cribl-to-splunk"
+
+  log_group_name = aws_cloudwatch_log_group.adot_log[0].name
+
+  filter_pattern  = ""
+  destination_arn = module.cribl_well_known.kinesis_destination_arns["eu-west-2"]
+  distribution    = "ByLogStream"
+  role_arn        = var.kinesis_subscription_role_arn
+}