Update DDE and storage modules

laurencegoolsby · laurencegoolsby · commit 436c674bf94d · 2026-01-22T22:44:22.000-05:00
- Add support for AWS-managed encryption (Bedrock Data Automation requires AWS-managed encryption)
- Add support for AWS-managed Bedrock Data Automation blueprints
diff --git a/infra/modules/document-data-extraction/resources/README.md b/infra/modules/document-data-extraction/resources/README.md
@@ -1,13 +1,20 @@
 # Bedrock Data Automation Terraform Module
 
-This module provisions AWS Bedrock Data Automation resources, including the data automation project, blueprints, and associated IAM role for accessing S3 buckets.
+This module provisions AWS Bedrock Data Automation resources, including the data automation project and blueprints.
+
 
 ## Overview
 
 The module creates:
 - **Bedrock Data Automation Project** - Main project resource for data automation workflows
 - **Bedrock Blueprints** - Custom extraction blueprints configured via a map
-- **IAM Role** - Role for Bedrock service to assume with access to input/output S3 buckets
+
+## Important Notes
+
+- **BDA uses its own internal service role** - This module does not create a custom IAM role for BDA. Bedrock Data Automation uses an AWS-managed internal service role for S3 access.
+- **S3 bucket encryption** - S3 buckets used with BDA should use AWS-managed encryption (AES256), not customer-managed KMS keys.
+- **Lambda permissions** - Any Lambda function invoking BDA must have S3 permissions for both input and output buckets directly attached to its execution role.
+- **No bucket policies needed** - BDA does not require bucket policies allowing the `bedrock.amazonaws.com` service principal.
 
 ## Features
 - Creates resources required for Bedrock Data Automation workflows
@@ -17,14 +24,49 @@ The module creates:
 - Complies with Checkov recommendations for security and compliance
 - Designed for cross-layer usage (see project module conventions)
 
+## Usage
+
+```hcl
+module "bedrock_data_automation" {
+  source = "../../modules/document-data-extraction/resources"
+  
+  name  = "my-app-prod"
+  
+  blueprints_map = {
+    invoice = {
+      schema = file("${path.module}/schemas/invoice.json")
+      type   = "DOCUMENT"
+      tags   = {
+          Environment = "production"
+          ManagedBy   = "terraform"
+      }
+    }
+  }
+  
+  standard_output_configuration = {
+    document = {
+      extraction = {
+        granularity = {
+          types = ["PAGE", "ELEMENT"]
+        }
+      }
+    }
+  }
+  
+  tags = {
+          Environment = "production"
+          ManagedBy   = "terraform"
+  }
+}
+```
+
 ## Inputs
 
 ### Required Variables
 
 | Name  | Description | Type | Required |
 |-------|-------------|------|----------|
 | `name` | Prefix to use for resource names (e.g., "my-app-prod") | `string` | yes |
-| `data_access_policy_arns` | Map of policy ARNs for input and output locations to attach to the BDA role | `map(string)` | yes |
 | `blueprints_map` | Map of unique blueprints with keys as blueprint identifiers and values as blueprint objects | `map(object)` | yes |
 
 #### `blueprints_map` Object Structure
@@ -59,17 +101,16 @@ See `variables.tf` for complete structure details.
 | Name | Description |
 |------|-------------|
 | `bda_project_arn` | The ARN of the Bedrock Data Automation project |
-| `bda_role_name` | The name of the IAM role used by Bedrock Data Automation |
-| `bda_role_arn` | The ARN of the IAM role used by Bedrock Data Automation |
 | `access_policy_arn` | The ARN of the IAM policy for accessing the Bedrock Data Automation project |
-
+| `bda_profile_arn` | The profile ARN for cross-region inference |
+| `bda_blueprint_arns` | List of created blueprint ARNs |
+| `bda_blueprint_names` | List of created blueprint names |
+| `bda_blueprint_arn_to_name` | Map of blueprint ARNs to names |
 
 ## Resources Created
 
 - `awscc_bedrock_data_automation_project.bda_project` - Main BDA project
 - `awscc_bedrock_blueprint.bda_blueprint` - One or more blueprints (created from `blueprints_map`)
-- `aws_iam_role.bda_role` - IAM role for Bedrock service
-- `aws_iam_role_policy_attachment.role_policy_attachments` - Policy attachments for S3 access
 
 ## Project Conventions
 
@@ -95,11 +136,6 @@ module "bedrock_data_automation" {
   
   name = "my-app"
   
-  data_access_policy_arns = {
-    input  = aws_iam_policy.input.arn
-    output = aws_iam_policy.output.arn
-  }
-  
   blueprints_map = {}  # No custom blueprints
 }
 ```
@@ -110,7 +146,6 @@ module "bedrock_data_automation" {
   source = "../../modules/document-data-extraction/resources"
   
   name               = "my-app"
-  data_access_policy_arns = { /* ... */ }
   blueprints_map     = { /* ... */ }
   
   standard_output_configuration = {
@@ -149,8 +184,6 @@ module "bedrock_data_automation" {
 - AWS provider configured
 - AWS Cloud Control provider (awscc) configured
 - Appropriate AWS permissions to create Bedrock and IAM resources
-- KMS keys
-- S3 bucket policies defined for input/output buckets
 
 ## References
 
diff --git a/infra/modules/document-data-extraction/resources/access_control.tf b/infra/modules/document-data-extraction/resources/access_control.tf
@@ -9,14 +9,16 @@ data "aws_iam_policy_document" "bedrock_access" {
       "bedrock:InvokeModel",
       "bedrock:InvokeModelWithResponseStream",
       "bedrock:GetDataAutomationProject",
+      "bedrock:GetBlueprint",
       "bedrock:StartDataAutomationJob",
       "bedrock:GetDataAutomationJob",
       "bedrock:ListDataAutomationJobs"
     ]
     effect = "Allow"
     resources = [
       awscc_bedrock_data_automation_project.bda_project.project_arn,
-      "${awscc_bedrock_data_automation_project.bda_project.project_arn}/*"
+      "${awscc_bedrock_data_automation_project.bda_project.project_arn}/*",
+      "arn:aws:bedrock:*:*:blueprint/*"
     ]
   }
 }
diff --git a/infra/modules/document-data-extraction/resources/encryption.tf b/infra/modules/document-data-extraction/resources/encryption.tf
diff --git a/infra/modules/document-data-extraction/resources/main.tf b/infra/modules/document-data-extraction/resources/main.tf
@@ -7,58 +7,38 @@ locals {
     }
   ]
 
-  kms_encryption_context = {
-    Environment = lookup(var.tags, "environment", "unknown")
-  }
+  all_blueprints = concat(
+    # custom blueprints created from json schemas
+    [for k, v in awscc_bedrock_blueprint.bda_blueprint : {
+      blueprint_arn   = v.blueprint_arn
+      blueprint_stage = v.blueprint_stage
+    }],
+    # aws managed blueprints referenced by arn
+    var.aws_managed_blueprints != null ? [
+      for arn in var.aws_managed_blueprints : {
+        blueprint_arn   = arn
+        blueprint_stage = "LIVE"
+      }
+    ] : []
+  )
 }
 
 resource "awscc_bedrock_data_automation_project" "bda_project" {
   project_name                  = "${var.name}-project"
   project_description           = "Project for ${var.name}"
-  kms_encryption_context        = local.kms_encryption_context
-  kms_key_id                    = aws_kms_key.bedrock_data_automation.arn
   tags                          = local.bda_tags
   standard_output_configuration = var.standard_output_configuration
-  custom_output_configuration = {
-    blueprints = [for k, v in awscc_bedrock_blueprint.bda_blueprint : {
-      blueprint_arn   = v.blueprint_arn
-      blueprint_stage = v.blueprint_stage
-    }]
-  }
+  custom_output_configuration = length(local.all_blueprints) > 0 ? {
+    blueprints = local.all_blueprints
+  } : null
   override_configuration = var.override_configuration
 }
 
 resource "awscc_bedrock_blueprint" "bda_blueprint" {
   for_each = var.blueprints_map
 
-  blueprint_name         = "${var.name}-${each.key}"
-  schema                 = each.value.schema
-  type                   = each.value.type
-  kms_encryption_context = local.kms_encryption_context
-  kms_key_id             = aws_kms_key.bedrock_data_automation.arn
-  tags                   = local.bda_tags
+  blueprint_name = "${var.name}-${each.key}"
+  schema         = each.value.schema
+  type           = each.value.type
+  tags           = local.bda_tags
 }
-
-resource "aws_iam_role" "bda_role" {
-  name = "${var.name}-bda_role"
-
-  assume_role_policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [
-      {
-        Effect = "Allow"
-        Principal = {
-          Service = "bedrock.amazonaws.com"
-        }
-        Action = "sts:AssumeRole"
-      }
-    ]
-  })
-}
-
-resource "aws_iam_role_policy_attachment" "role_policy_attachments" {
-  for_each = var.data_access_policy_arns
-
-  role       = aws_iam_role.bda_role.name
-  policy_arn = each.value
-}
diff --git a/infra/modules/document-data-extraction/resources/variables.tf b/infra/modules/document-data-extraction/resources/variables.tf
@@ -3,9 +3,10 @@ variable "name" {
   type        = string
 }
 
-variable "data_access_policy_arns" {
-  description = "The set of policy ARNs for the input and output locations to attach to the BDA role."
-  type        = map(string)
+variable "aws_managed_blueprints" {
+  description = "List of AWS managed blueprint ARNs (stage defaults to LIVE)"
+  type        = list(string)
+  default     = null
 }
 
 variable "custom_output_config" {
diff --git a/infra/modules/storage/access_control.tf b/infra/modules/storage/access_control.tf
@@ -57,9 +57,12 @@ data "aws_iam_policy_document" "storage_access" {
       "arn:aws:s3:::${var.name}/*"
     ]
   }
-  statement {
-    actions   = ["kms:GenerateDataKey", "kms:Decrypt"]
-    effect    = "Allow"
-    resources = [aws_kms_key.storage.arn]
+  dynamic "statement" {
+    for_each = var.use_aws_managed_encryption ? [] : [1]
+    content {
+      actions   = ["kms:GenerateDataKey", "kms:Decrypt"]
+      effect    = "Allow"
+      resources = [aws_kms_key.storage[0].arn]
+    }
   }
 }
diff --git a/infra/modules/storage/encryption.tf b/infra/modules/storage/encryption.tf
@@ -1,4 +1,6 @@
 resource "aws_kms_key" "storage" {
+  count = var.use_aws_managed_encryption ? 0 : 1
+
   description = "KMS key for bucket ${var.name}"
   # The waiting period, specified in number of days. After the waiting period ends, AWS KMS deletes the KMS key.
   deletion_window_in_days = "10"
@@ -10,8 +12,8 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "storage" {
   bucket = aws_s3_bucket.storage.id
   rule {
     apply_server_side_encryption_by_default {
-      kms_master_key_id = aws_kms_key.storage.arn
-      sse_algorithm     = "aws:kms"
+      kms_master_key_id = var.use_aws_managed_encryption ? null : aws_kms_key.storage[0].arn
+      sse_algorithm     = var.use_aws_managed_encryption ? "AES256" : "aws:kms"
     }
     bucket_key_enabled = true
   }
diff --git a/infra/modules/storage/main.tf b/infra/modules/storage/main.tf
@@ -9,4 +9,5 @@ resource "aws_s3_bucket" "storage" {
   # checkov:skip=CKV_AWS_144:Cross region replication not required by default
   # checkov:skip=CKV2_AWS_62:S3 bucket does not need notifications enabled
   # checkov:skip=CKV_AWS_21:Bucket versioning is not needed
+  # checkov:skip=CKV_AWS_145:AWS-managed encryption (AES256) used when use_aws_managed_encryption=true
 }
diff --git a/infra/modules/storage/variables.tf b/infra/modules/storage/variables.tf
@@ -8,3 +8,9 @@ variable "name" {
   type        = string
   description = "Name of the AWS S3 bucket. Needs to be globally unique across all regions."
 }
+
+variable "use_aws_managed_encryption" {
+  description = "Use AWS-managed encryption (AES256) instead of customer-managed KMS keys"
+  type        = bool
+  default     = false
+}
diff --git a/infra/{{app_name}}/app-config/env-config/document_data_extraction.tf b/infra/{{app_name}}/app-config/env-config/document_data_extraction.tf
@@ -7,7 +7,10 @@ locals {
     # Blueprints path is relative to infra/{{app_name}}/service/ directory
     # Contains JSON schema files for custom Bedrock Data Automation blueprints
     # (e.g., not AWS-managed blueprints)
-    blueprints_path    = "./document-data-extraction-blueprints/"
+    blueprints_path = "./document-data-extraction-blueprints/"
+
+    # Optional: List of AWS-managed blueprint ARNs to use for document extraction
+    aws_managed_blueprints = null
 
     # BDA can only be deployed to us-east-1, us-west-2, and us-gov-west-1
     # TODO(https://github.com/navapbc/template-infra/issues/993) Add GovCloud Support
diff --git a/infra/{{app_name}}/service/document_data_extraction.tf b/infra/{{app_name}}/service/document_data_extraction.tf
@@ -24,21 +24,23 @@ module "dde_input_bucket" {
     aws = aws.dde
   }
 
-  count        = local.document_data_extraction_config != null ? 1 : 0
-  source       = "../../modules/storage"
-  name         = "${local.prefix}${local.document_data_extraction_config.input_bucket_name}"
-  is_temporary = local.is_temporary
+  count                      = local.document_data_extraction_config != null ? 1 : 0
+  source                     = "../../modules/storage"
+  name                       = "${local.prefix}${local.document_data_extraction_config.input_bucket_name}"
+  is_temporary               = local.is_temporary
+  use_aws_managed_encryption = true
 }
 
 module "dde_output_bucket" {
   providers = {
     aws = aws.dde
   }
 
-  count        = local.document_data_extraction_config != null ? 1 : 0
-  source       = "../../modules/storage"
-  name         = "${local.prefix}${local.document_data_extraction_config.output_bucket_name}"
-  is_temporary = local.is_temporary
+  count                      = local.document_data_extraction_config != null ? 1 : 0
+  source                     = "../../modules/storage"
+  name                       = "${local.prefix}${local.document_data_extraction_config.output_bucket_name}"
+  is_temporary               = local.is_temporary
+  use_aws_managed_encryption = true
 }
 
 module "dde" {
@@ -51,6 +53,7 @@ module "dde" {
   source = "../../modules/document-data-extraction/resources"
 
   standard_output_configuration = local.document_data_extraction_config.standard_output_configuration
+  aws_managed_blueprints        = local.document_data_extraction_config.aws_managed_blueprints
   tags                          = local.tags
 
   blueprints_map = {
@@ -65,9 +68,4 @@ module "dde" {
   }
 
   name = "${local.prefix}${local.document_data_extraction_config.name}"
-
-  data_access_policy_arns = {
-    input_bucket  = module.dde_input_bucket[0].access_policy_arn,
-    output_bucket = module.dde_output_bucket[0].access_policy_arn
-  }
 }

Original file line number	Diff line number	Diff line change
`@@ -9,14 +9,16 @@ data "aws_iam_policy_document" "bedrock_access" {`
`9`	`9`	`"bedrock:InvokeModel",`
`10`	`10`	`"bedrock:InvokeModelWithResponseStream",`
`11`	`11`	`"bedrock:GetDataAutomationProject",`
	`12`	`+ "bedrock:GetBlueprint",`
`12`	`13`	`"bedrock:StartDataAutomationJob",`
`13`	`14`	`"bedrock:GetDataAutomationJob",`
`14`	`15`	`"bedrock:ListDataAutomationJobs"`
`15`	`16`	`]`
`16`	`17`	`effect = "Allow"`
`17`	`18`	`resources = [`
`18`	`19`	`awscc_bedrock_data_automation_project.bda_project.project_arn,`
`19`		`- "${awscc_bedrock_data_automation_project.bda_project.project_arn}/*"`
	`20`	`+ "${awscc_bedrock_data_automation_project.bda_project.project_arn}/*",`
	`21`	`+ "arn:aws:bedrock:::blueprint/*"`
`20`	`22`	`]`
`21`	`23`	`}`
`22`	`24`	`}`
Original file line number	Diff line number	Diff line change
`@@ -3,9 +3,10 @@ variable "name" {`
`3`	`3`	`type = string`
`4`	`4`	`}`
`5`	`5`
`6`		`-variable "data_access_policy_arns" {`
`7`		`- description = "The set of policy ARNs for the input and output locations to attach to the BDA role."`
`8`		`- type = map(string)`
	`6`	`+variable "aws_managed_blueprints" {`
	`7`	`+ description = "List of AWS managed blueprint ARNs (stage defaults to LIVE)"`
	`8`	`+ type = list(string)`
	`9`	`+ default = null`
`9`	`10`	`}`
`10`	`11`
`11`	`12`	`variable "custom_output_config" {`
Original file line number	Diff line number	Diff line change
`@@ -57,9 +57,12 @@ data "aws_iam_policy_document" "storage_access" {`
`57`	`57`	`"arn:aws:s3:::${var.name}/*"`
`58`	`58`	`]`
`59`	`59`	`}`
`60`		`- statement {`
`61`		`- actions = ["kms:GenerateDataKey", "kms:Decrypt"]`
`62`		`- effect = "Allow"`
`63`		`- resources = [aws_kms_key.storage.arn]`
	`60`	`+ dynamic "statement" {`
	`61`	`+ for_each = var.use_aws_managed_encryption ? [] : [1]`
	`62`	`+ content {`
	`63`	`+ actions = ["kms:GenerateDataKey", "kms:Decrypt"]`
	`64`	`+ effect = "Allow"`
	`65`	`+ resources = [aws_kms_key.storage[0].arn]`
	`66`	`+ }`
`64`	`67`	`}`
`65`	`68`	`}`
Original file line number	Diff line number	Diff line change
`@@ -9,4 +9,5 @@ resource "aws_s3_bucket" "storage" {`
`9`	`9`	`# checkov:skip=CKV_AWS_144:Cross region replication not required by default`
`10`	`10`	`# checkov:skip=CKV2_AWS_62:S3 bucket does not need notifications enabled`
`11`	`11`	`# checkov:skip=CKV_AWS_21:Bucket versioning is not needed`
	`12`	`+ # checkov:skip=CKV_AWS_145:AWS-managed encryption (AES256) used when use_aws_managed_encryption=true`
`12`	`13`	`}`