Wbprice/troubleshoot api service not starting (#121)

* Troubleshoot API Startup Failures

- Troubleshoot networking configuration between load balancer and API
  security group
- Troubleshoot API IAM permissions
- Troubleshoot vpn security group permission
- add env.* to gitignore

---------

Co-authored-by: Blaine Price <william.price@cms.hhs.gov>

Author: wbprice

infrastructure/.gitignore

@@ -1,4 +1,5 @@
 .terraform
 .terraform.lock.hcl
 terraform.tfstate
-terraform.tfstate.backup
+terraform.tfstate.backup
+.env.*

infrastructure/envs/dev/main.tf

@@ -4,7 +4,7 @@ terraform {
   required_providers {
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 5.0"
+      version = "~> 6.0"
     }
   }
 
@@ -51,6 +51,7 @@ module "api-db" {
   allocated_storage       = 20
   publicly_accessible     = false
   username                = "npd"
+  db_name                 = "npd"
   vpc_security_group_ids  = [module.networking.db_security_group_id]
   db_subnet_group_name    = module.networking.db_subnet_group_name
   backup_retention_period = 7             # Remove automated snapshots after 7 days
@@ -79,21 +80,16 @@ module "etl-db" {
 # ECS Cluster
 module "ecs" {
   source  = "terraform-aws-modules/ecs/aws"
-  version = "5.12.1"
+  version = "6.6.2"
 
   cluster_name = "${local.account_name}-ecs-cluster"
-
-  fargate_capacity_providers = {
+  default_capacity_provider_strategy = {
     FARGATE = {
-      default_capacity_provider_strategy = {
-        weight = 50
-        base   = 20
-      }
+      weight = 50
+      base   = 20
     }
     FARGATE_SPOT = {
-      default_capacity_provider_strategy = {
-        weight = 50
-      }
+      weight = 50
     }
   }
 }
@@ -103,14 +99,14 @@ module "fhir-api" {
   source = "../../modules/fhir-api"
 
   account_name             = local.account_name
-  app_db_name              = "npd"
   fhir_api_migration_image = var.migration_image
   fhir_api_image           = var.fhir_api_image
   ecs_cluster_id           = module.ecs.cluster_id
   db = {
     db_instance_master_user_secret_arn = module.api-db.db_instance_master_user_secret_arn
     db_instance_address                = module.api-db.db_instance_address
     db_instance_port                   = module.api-db.db_instance_port
+    db_instance_name                   = module.api-db.db_instance_name
   }
   networking = {
     db_subnet_ids         = module.networking.db_subnet_ids

infrastructure/envs/dev/variables.tf

@@ -6,5 +6,5 @@ variable "tier" {
   default = "dev"
 }
 
-variable "migration_image" { default = "public.ecr.aws/docker/library/hello-world:nanoserver-ltsc2022" }
-variable "fhir_api_image" { default = "public.ecr.aws/docker/library/hello-world:nanoserver-ltsc2022" }
+variable "migration_image" { default = "575012135727.dkr.ecr.us-east-1.amazonaws.com/npd-east-dev-fhir-api-migrations:latest" }
+variable "fhir_api_image" { default = "575012135727.dkr.ecr.us-east-1.amazonaws.com/npd-east-dev-fhir-api:latest" }

infrastructure/modules/fhir-api/main.tf

@@ -3,6 +3,7 @@ data "aws_partition" "current" {}
 data "aws_caller_identity" "current" {}
 
 # ECR Repositories
+
 resource "aws_ecr_repository" "fhir_api" {
   name = "${var.account_name}-fhir-api"
 }
@@ -11,10 +12,22 @@ resource "aws_ecr_repository" "fhir_api_migrations" {
   name = "${var.account_name}-fhir-api-migrations"
 }
 
+# Log Groups
+
+resource "aws_cloudwatch_log_group" "fhir_api_log_group" {
+  name              = "/ecs/${var.account_name}-fhir-api-logs"
+  retention_in_days = 30
+}
+
+resource "aws_cloudwatch_log_group" "fhir_api_migrations_log_group" {
+  name              = "/ecs/${var.account_name}-fhir-api-migrations-logs"
+  retention_in_days = 30
+}
+
 # ECS Roles and Policies
-resource "aws_iam_role" "fhir_api_role" {
-  name        = "${var.account_name}-fhir-api-role"
-  description = "Defines what AWS actions the FHIR API task is allowed to make"
+resource "aws_iam_role" "fhir_api_execution_role" {
+  name        = "${var.account_name}-fhir-api-execution-role"
+  description = "Defines what AWS actions the FHIR API task execution environment is allowed to make"
   assume_role_policy = jsonencode({
     Version = "2012-10-17"
     Statement = [{
@@ -25,6 +38,11 @@ resource "aws_iam_role" "fhir_api_role" {
   })
 }
 
+resource "aws_iam_role_policy_attachment" "ecs_task_execution" {
+  role       = aws_iam_role.fhir_api_execution_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
+}
+
 resource "aws_iam_policy" "fhir_api_can_access_fhir_api_db_secret" {
   name        = "${var.account_name}-fhir-api-can-access-fhir-database-secret"
   description = "Allows ECS to access the RDS secret"
@@ -36,22 +54,21 @@ resource "aws_iam_policy" "fhir_api_can_access_fhir_api_db_secret" {
         Effect = "Allow"
         Resource = [
           var.db.db_instance_master_user_secret_arn,
-          aws_secretsmanager_secret.django_secret.arn
+          aws_secretsmanager_secret_version.django_secret_version.arn
         ]
       }
     ]
   })
 }
 
 resource "aws_iam_role_policy_attachment" "fhir_api_can_access_database_secret_attachment" {
-  role       = aws_iam_role.fhir_api_role.name
+  role       = aws_iam_role.fhir_api_execution_role.name
   policy_arn = aws_iam_policy.fhir_api_can_access_fhir_api_db_secret.arn
 }
 
 resource "aws_iam_policy" "fhir_api_logs_policy" {
   name        = "${var.account_name}-fhir-api-can-log-to-cloudwatch"
   description = "Allow ECS tasks to write logs to CloudWatch"
-  path        = "/delegatedadmin/developer/"
 
   policy = jsonencode({
     Version = "2012-10-17"
@@ -61,16 +78,18 @@ resource "aws_iam_policy" "fhir_api_logs_policy" {
           "logs:CreateLogStream",
           "logs:PutLogEvents"
         ]
-        Effect   = "Allow"
-        Resource = "arn:${data.aws_partition.current.partition}:logs:*:${data.aws_caller_identity.current.account_id}:log-group:/ecs/${var.account_name}*:*"
+        Effect = "Allow"
+        Resource = [
+          "arn:${data.aws_partition.current.partition}:logs:*:${data.aws_caller_identity.current.account_id}:log-group:/ecs/${var.account_name}*:*"
+        ]
       },
     ]
   })
 }
 
 resource "aws_iam_role_policy_attachment" "fhir_api_can_create_cloudwatch_logs" {
   policy_arn = aws_iam_policy.fhir_api_logs_policy.id
-  role       = aws_iam_role.fhir_api_role.id
+  role       = aws_iam_role.fhir_api_execution_role.id
 }
 
 # FHIR API Secrets
@@ -96,7 +115,7 @@ resource "aws_ecs_task_definition" "app" {
   network_mode             = "awsvpc"
   cpu                      = "512"
   memory                   = "1024"
-  execution_role_arn       = aws_iam_role.fhir_api_role.arn
+  execution_role_arn       = aws_iam_role.fhir_api_execution_role.arn
 
   container_definitions = jsonencode([
     # In the past, I've put the migration container in a separate task and invoked it manually to avoid the case
@@ -113,7 +132,7 @@ resource "aws_ecs_task_definition" "app" {
       environment = [
         {
           name  = "FLYWAY_URL"
-          value = "jdbc:postgresql://${var.db.db_instance_address}:${var.db.db_instance_port}/${var.app_db_name}"
+          value = "jdbc:postgresql://${var.db.db_instance_address}:${var.db.db_instance_port}/${var.db.db_instance_name}"
         }
       ],
       secrets = [
@@ -129,9 +148,9 @@ resource "aws_ecs_task_definition" "app" {
       logConfiguration = {
         logDriver = "awslogs"
         options = {
-          "awslogs-group"         = "/ecs/${var.account_name}-fhir-api-migration-logs"
-          "awslogs-region"        = data.aws_region.current.name
-          "awslogs-stream-prefix" = "${var.account_name}-fhir-api-migration-logs"
+          "awslogs-group"         = aws_cloudwatch_log_group.fhir_api_migrations_log_group.name
+          "awslogs-region"        = "us-east-1"
+          "awslogs-stream-prefix" = var.account_name
         }
       }
     },
@@ -142,7 +161,7 @@ resource "aws_ecs_task_definition" "app" {
       environment = [
         {
           name  = "NPD_DB_NAME"
-          value = var.app_db_name
+          value = var.db.db_instance_name
         },
         {
           name  = "NPD_DB_HOST"
@@ -170,7 +189,7 @@ resource "aws_ecs_task_definition" "app" {
         },
         {
           name  = "NPD_PROJECT_NAME"
-          value = "ndh"
+          value = "npd"
         },
         {
           name  = "CACHE_LOCATION",
@@ -191,13 +210,15 @@ resource "aws_ecs_task_definition" "app" {
           valueFrom = "${var.db.db_instance_master_user_secret_arn}:password::"
         },
       ]
-      portMappings = [{ containerPort = var.fhir_api_port }]
+      portMappings = [{
+        containerPort = var.fhir_api_port
+      }]
       logConfiguration = {
         logDriver = "awslogs"
         options = {
-          "awslogs-group"         = "/ecs/${var.account_name}-fhir-api-logs"
-          "awslogs-region"        = data.aws_region.current.name
-          "awslogs-stream-prefix" = "${var.account_name}-fhir-api-logs"
+          "awslogs-group"         = aws_cloudwatch_log_group.fhir_api_log_group.name
+          "awslogs-region"        = "us-east-1"
+          "awslogs-stream-prefix" = var.account_name
         }
       }
       #   TODO: Implement for your app
@@ -227,7 +248,7 @@ resource "aws_ecs_service" "app" {
   }
 
   load_balancer {
-    target_group_arn = aws_lb_target_group.fhir_api.arn
+    target_group_arn = aws_lb_target_group.fhir_api_tg.arn
     container_name   = "${var.account_name}-fhir-api"
     container_port   = var.fhir_api_port
   }
@@ -242,7 +263,7 @@ resource "aws_lb" "fhir_api_alb" {
   subnets            = var.networking.public_subnet_ids
 }
 
-resource "aws_lb_target_group" "fhir_api" {
+resource "aws_lb_target_group" "fhir_api_tg" {
   name        = "${var.account_name}-fhir-api-tg"
   port        = var.fhir_api_port
   protocol    = "HTTP"
@@ -256,7 +277,7 @@ resource "aws_lb_target_group" "fhir_api" {
     timeout             = 5
     healthy_threshold   = 2
     unhealthy_threshold = 10
-    matcher = "200"
+    matcher             = "200"
   }
 }
 
@@ -267,6 +288,6 @@ resource "aws_lb_listener" "http" {
 
   default_action {
     type             = "forward"
-    target_group_arn = aws_lb_target_group.fhir_api.arn
+    target_group_arn = aws_lb_target_group.fhir_api_tg.arn
   }
 }

infrastructure/modules/fhir-api/variables.tf

@@ -1,15 +1,15 @@
 variable "account_name" {}
-variable "app_db_name" {}
 variable "fhir_api_image" {}
+variable "fhir_api_migration_image" {}
 variable "fhir_api_port" {
   default = 8000
 }
-variable "fhir_api_migration_image" {}
 variable "ecs_cluster_id" {}
 variable "db" {
   type = object({
     db_instance_master_user_secret_arn = string
     db_instance_address                = string
+    db_instance_name                   = string
     db_instance_port                   = string
   })
 }