Merge pull request #200 from nationalarchives/bulkdownload/infra

Vaishnavir9901 · web-flow · commit 139f49e46198 · 2026-06-26T15:59:22.000+01:00
Bulk download infra
diff --git a/.github/ansible-playbooks/bulkdownload/docker/compose.yml.template b/.github/ansible-playbooks/bulkdownload/docker/compose.yml.template
@@ -2,7 +2,7 @@ services:
   blue-web: &bulkdownload
     container_name: blue-web
     env_file: /var/docker/bulkdownload.env
-    image: "ghcr.io/nationalarchives/ds-frontend-bulkdownload:25.06.02.37"
+    image: "ghcr.io/nationalarchives/ds-bulk-download:25.06.02.37"
     labels:
       - "traefik.enable=true"
       - "traefik.docker.network=traefik_webgateway"
diff --git a/.github/ansible-playbooks/bulkdownload/templates/compose.yml.j2 b/.github/ansible-playbooks/bulkdownload/templates/compose.yml.j2
@@ -2,7 +2,7 @@ services:
   blue-web: &bulkdownload
     container_name: blue-web
     env_file: /var/docker/bulkdownload.env
-    image: "ghcr.io/nationalarchives/ds-bulkdownload:26.06.26.36"
+    image: "ghcr.io/nationalarchives/ds-bulk-download:26.06.26.36"
     labels:
       - "traefik.enable=true"
       - "traefik.docker.network='traefik_webgateway'"
diff --git a/components/terraform/_data-ami.tf b/components/terraform/_data-ami.tf
@@ -253,6 +253,7 @@ data "aws_ami" "web_hosprec_ami" {
 }
 
 data "aws_ami" "web_bulkdownload_ami" {
+    count = var.environment == "live" ? 1 : 0
     most_recent = true
 
     filter {
diff --git a/components/terraform/_vars.tf b/components/terraform/_vars.tf
@@ -10,7 +10,6 @@ variable "web_frontend_logfile_s3_root" {}
 variable "public_domain" {}
 variable "region" {}
 variable "db_record" {}
-variable "cf_dist" {}
 variable "internal_domain_name" {}
 variable "web_enrichment_efs_mount_dir" {}
 variable "web_frontend_efs_mount_dir" {}
diff --git a/components/terraform/_vars/dev/autoscalinggroup.auto.tfvars b/components/terraform/_vars/dev/autoscalinggroup.auto.tfvars
@@ -128,3 +128,13 @@ web_hosprec_asg_min_size                  = 1
 web_hosprec_asg_desired_capacity          = 1
 web_hosprec_asg_health_check_grace_period = 150
 web_hosprec_asg_health_check_type         = "EC2"
+
+# Bulkdownload
+web_bulkdownload_deployment_group              = ""
+web_bulkdownload_patch_group                   = ""
+
+web_bulkdownload_asg_max_size                  = 0
+web_bulkdownload_asg_min_size                  = 0
+web_bulkdownload_asg_desired_capacity          = 0
+web_bulkdownload_asg_health_check_grace_period = 150
+web_bulkdownload_asg_health_check_type         = "EC2"
diff --git a/components/terraform/_vars/dev/bulkdownload.auto.tfvars b/components/terraform/_vars/dev/bulkdownload.auto.tfvars
@@ -0,0 +1,10 @@
+web_bulkdownload_deployment_s3_bucket = ""
+web_bulkdownload_folder_s3_key = ""
+
+web_bulkdownload_key_name      = ""
+web_bulkdownload_instance_type = ""
+
+web_bulkdownload_auto_switch_on   = false
+web_bulkdownload_auto_switch_off  = false
+
+web_bulkdownload_root_block_device_size = "100"
diff --git a/components/terraform/_vars/dev/cloudfront.auto.tfvars b/components/terraform/_vars/dev/cloudfront.auto.tfvars
@@ -1,21 +1,3 @@
-cf_dist = {
-    "create_distribution"                            = true
-    "cfd_domain_name"                                = "internal-web-frontend-1356619265.eu-west-2.elb.amazonaws.com"
-    "cfd_origin_id"                                  = "dev-www.nationalarchives.gov.uk"
-    "cfd_price_class"                                = "PriceClass_100"
-    "cfd_enabled"                                    = true
-    "cfd_aliases"                                    = ["dev-www.nationalarchives.gov.uk"]
-    "cfd_default_behaviour_allowed_methods"          = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
-    "cfd_default_behaviour_cached_methods"           = ["GET", "HEAD"]
-    "cfd_behaviour_default_viewer_protocol_policy"   = "redirect-to-https"
-    "cfd_behaviour_viewer_protocol_policy"           = "redirect-to-https"
-    "cfd_behaviour_compress"                         = true
-    "cfd_cache_disabled_path_patterns"               = []
-    "cfd_Managed_CachingOptimized_cache_policy_id"   = "2e54312d-136d-493c-8eb9-b001f22f67d2"
-    "cfd_Managed_AllViewer_origin_request_policy_id" = "216adef6-5c7f-47e4-b989-5492eafa07d3"
-    "cfd_Managed_CachingDisabled_cache_policy_id"    = "4135ea2d-6df8-44a3-9df3-4b5a84be39ad"
-}
-
 cf_default_ttl = 300
 cf_min_ttl     = 60
 cf_max_ttl     = 1200
diff --git a/components/terraform/_vars/live/autoscalinggroup.auto.tfvars b/components/terraform/_vars/live/autoscalinggroup.auto.tfvars
@@ -123,8 +123,18 @@ web_reverse_proxy_scale_out_threshold            = 0
 web_hosprec_patch_group      = "web-hosprec-patchgroup"
 web_hosprec_deployment_group = "web-hosprec-deploygroup"
 
-web_hosprec_asg_max_size                  = 2
-web_hosprec_asg_min_size                  = 1
-web_hosprec_asg_desired_capacity          = 1
+web_hosprec_asg_max_size                  = 3
+web_hosprec_asg_min_size                  = 2
+web_hosprec_asg_desired_capacity          = 2
 web_hosprec_asg_health_check_grace_period = 150
 web_hosprec_asg_health_check_type         = "EC2"
+
+# Bulkdownload
+web_bulkdownload_deployment_group              = "web-bulkdownload-patchgroup"
+web_bulkdownload_patch_group                   = "web-bulkdownload-deploygroup"
+
+web_bulkdownload_asg_max_size                  = 2
+web_bulkdownload_asg_min_size                  = 1
+web_bulkdownload_asg_desired_capacity          = 1
+web_bulkdownload_asg_health_check_grace_period = 150
+web_bulkdownload_asg_health_check_type         = "EC2"
diff --git a/components/terraform/_vars/live/bulkdownload.auto.tfvars b/components/terraform/_vars/live/bulkdownload.auto.tfvars
@@ -0,0 +1,10 @@
+web_bulkdownload_deployment_s3_bucket = "ds-live-deployment-source"
+web_bulkdownload_folder_s3_key = "web/bulkdownload"
+
+web_bulkdownload_key_name      = "web-frontend-live-eu-west-2"
+web_bulkdownload_instance_type = "t3a.medium"
+
+web_bulkdownload_auto_switch_on   = true
+web_bulkdownload_auto_switch_off  = true
+
+web_bulkdownload_root_block_device_size = "100"
diff --git a/components/terraform/_vars/live/cloudfront.auto.tfvars b/components/terraform/_vars/live/cloudfront.auto.tfvars
@@ -1,21 +1,3 @@
-cf_dist = {
-    "create_distribution"                            = true
-    "cfd_domain_name"                                = "internal-web-frontend-360074626.eu-west-2.elb.amazonaws.com"
-    "cfd_origin_id"                                  = "www.nationalarchives.gov.uk"
-    "cfd_price_class"                                = "PriceClass_100"
-    "cfd_enabled"                                    = true
-    "cfd_aliases"                                    = ["www.nationalarchives.gov.uk"]
-    "cfd_default_behaviour_allowed_methods"          = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
-    "cfd_default_behaviour_cached_methods"           = ["GET", "HEAD"]
-    "cfd_behaviour_default_viewer_protocol_policy"   = "redirect-to-https"
-    "cfd_behaviour_viewer_protocol_policy"           = "redirect-to-https"
-    "cfd_behaviour_compress"                         = true
-    "cfd_cache_disabled_path_patterns"               = []
-    "cfd_Managed_CachingOptimized_cache_policy_id"   = "2e54312d-136d-493c-8eb9-b001f22f67d2"
-    "cfd_Managed_AllViewer_origin_request_policy_id" = "216adef6-5c7f-47e4-b989-5492eafa07d3"
-    "cfd_Managed_CachingDisabled_cache_policy_id"    = "4135ea2d-6df8-44a3-9df3-4b5a84be39ad"
-}
-
 cf_default_ttl = 300
 cf_min_ttl     = 60
 cf_max_ttl     = 1200
diff --git a/components/terraform/_vars/live/environment.auto.tfvars b/components/terraform/_vars/live/environment.auto.tfvars
@@ -2,11 +2,7 @@
 # Environment Specific Variables
 # -------------------------------------------------------------------------------------------------------------------
 
-owner          = "Digital Services"
-cost_centre    = "53"
 environment    = "live"
-created_by     = "Digital Services Platform Team"
-location       = "eu-west-2"
 account        = "live"
 
 default_tags = {
diff --git a/components/terraform/_vars/live/route53.auto.tfvars b/components/terraform/_vars/live/route53.auto.tfvars
@@ -1,3 +1,7 @@
 public_domain = "www.nationalarchives.gov.uk"
 private_record = "web-frontend.live.local"
 db_record = "web-frontend-db.live.local"
+create_wp_route53_health_check = true
+
+create_blog_zone  = true
+create_media_zone = true
diff --git a/components/terraform/_vars/staging/cloudfront.auto.tfvars b/components/terraform/_vars/staging/cloudfront.auto.tfvars
@@ -1,17 +1,13 @@
-cf_dist = {
-    "create_distribution"                            = true
-    "cfd_domain_name"                                = "internal-web-frontend-479883459.eu-west-2.elb.amazonaws.com"
-    "cfd_origin_id"                                  = "staging-www.nationalarchives.gov.uk"
-    "cfd_price_class"                                = "PriceClass_100"
-    "cfd_enabled"                                    = true
-    "cfd_aliases"                                    = ["staging-www.nationalarchives.gov.uk"]
-    "cfd_default_behaviour_allowed_methods"          = ["DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"]
-    "cfd_default_behaviour_cached_methods"           = ["GET", "HEAD"]
-    "cfd_behaviour_default_viewer_protocol_policy"   = "redirect-to-https"
-    "cfd_behaviour_viewer_protocol_policy"           = "redirect-to-https"
-    "cfd_behaviour_compress"                         = true
-    "cfd_cache_disabled_path_patterns"               = []
-    "cfd_Managed_CachingOptimized_cache_policy_id"   = "2e54312d-136d-493c-8eb9-b001f22f67d2"
-    "cfd_Managed_AllViewer_origin_request_policy_id" = "216adef6-5c7f-47e4-b989-5492eafa07d3"
-    "cfd_Managed_CachingDisabled_cache_policy_id"    = "4135ea2d-6df8-44a3-9df3-4b5a84be39ad"
-}
+cf_default_ttl = 60
+cf_min_ttl     = 150
+cf_max_ttl     = 30
+
+#
+# policy TTLs
+help_with_your_research_default_ttl = 3600
+help_with_your_research_max_ttl = 86400
+help_with_your_research_min_ttl = 300
+
+web_5_minute_update_default_ttl = 300
+web_5_minute_update_max_ttl = 1200
+web_5_minute_update_min_ttl = 0
diff --git a/components/terraform/iam/roles/outputs.tf b/components/terraform/iam/roles/outputs.tf
@@ -146,4 +146,13 @@ output "web_request_service_record_role_name" {
 
 output "web_request_service_record_instance_profile_arn"{
   value = aws_iam_instance_profile.web_request_service_record_profile.arn
-}
+}
+
+## Bulkdownload
+output "web_bulkdownload_role_name" {
+  value = var.environment == "live" ? aws_iam_role.web_bulkdownload_role[0].name : null
+}
+
+output "web_bulkdownload_instance_profile_arn" {
+  value = var.environment == "live" ? aws_iam_instance_profile.web_bulkdownload_profile[0].arn : null
+}
diff --git a/components/terraform/iam/roles/roles.tf b/components/terraform/iam/roles/roles.tf
@@ -58,6 +58,13 @@ resource "aws_iam_role" "web_reverse_proxy_role" {
     assume_role_policy = file("${path.root}/shared-templates/ec2_assume_role.json")
 }
 
+# Web Bulkdownload Role
+resource "aws_iam_role" "web_bulkdownload_role" {
+    count = var.environment == "live" ? 1 : 0
+    name = "web-bulkdownload-role"
+    assume_role_policy = file("${path.root}/shared-templates/ec2_assume_role.json")
+}
+
 # Request Service Record Role - MoD FoI
 resource "aws_iam_role" "web_request_service_record_role" {
   name = "web-request-service-record-role"
@@ -191,6 +198,14 @@ resource "aws_iam_instance_profile" "web_catalogue_profile" {
     role = aws_iam_role.web_catalogue_role.name
 }
 
+## Instance Profile for Web Bulkdownload Role
+resource "aws_iam_instance_profile" "web_bulkdownload_profile" {
+  count = var.environment == "live" ? 1 : 0
+
+  name = "web-bulkdownload-profile"
+  role = aws_iam_role.web_bulkdownload_role[0].name
+}
+
 #---------------------------------------------------------------------------
 ################### Policies ##########################################
 #---------------------------------------------------------------------------
@@ -554,3 +569,44 @@ resource "aws_iam_role_policy_attachment" "codedeploy_web_lambda_sync_s3_to_efs_
 #     role       = aws_iam_role.codedeploy_web_service_role.name
 #     policy_arn = var.s3_deployment_source_static_content_read_arn
 # }
+
+##-------------------------------------------------------------  
+####### Attach Policies to Web Bulk Download Role
+##-------------------------------------------------------------
+
+resource "aws_iam_role_policy_attachment" "web_bulkdownload_policy_attachment_1" {
+  count      = var.environment == "live" ? 1 : 0
+  role       = aws_iam_role.web_bulkdownload_role[0].name
+  policy_arn = "arn:aws:iam::aws:policy/CloudWatchAgentServerPolicy"
+}
+
+resource "aws_iam_role_policy_attachment" "web_bulkdownload_policy_attachment_2" {
+  count      = var.environment == "live" ? 1 : 0
+  role       = aws_iam_role.web_bulkdownload_role[0].name
+  policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"
+}
+
+resource "aws_iam_role_policy_attachment" "web_bulkdownload_policy_attachment_3" {
+  count      = var.environment == "live" ? 1 : 0
+  role       = aws_iam_role.web_bulkdownload_role[0].name
+  policy_arn = var.org_level_logging_arn
+}
+
+resource "aws_iam_role_policy_attachment" "web_bulkdownload_policy_attachment_4" {
+  count      = var.environment == "live" ? 1 : 0
+  role       = aws_iam_role.web_bulkdownload_role[0].name
+  policy_arn = var.deployment_s3_policy
+}
+
+resource "aws_iam_role_policy_attachment" "web_bulkdownload_policy_attachment_5" {
+  count      = var.environment == "live" ? 1 : 0
+  role       = aws_iam_role.web_bulkdownload_role[0].name
+  policy_arn = var.application_parameter_store_policy_arn
+}
+
+resource "aws_iam_role_policy_attachment" "web_bulkdownload_policy_attachment_6" {
+  count      = var.environment == "live" ? 1 : 0
+  role       = aws_iam_role.web_bulkdownload_role[0].name
+  policy_arn = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/org-session-manager-logs"
+}
+
diff --git a/components/terraform/security-groups.tf b/components/terraform/security-groups.tf
@@ -3,7 +3,7 @@ module "sgs" {
     
     vpc_id = data.aws_ssm_parameter.vpc_id.value
     lb_security_group_id = module.load-balancer.lb_security_group_id
-    
+    environment = var.environment
 
     instance_cidr = [
         data.aws_ssm_parameter.private_subnet_2a_cidr.value,
diff --git a/components/terraform/security-groups/outputs.tf b/components/terraform/security-groups/outputs.tf
@@ -43,3 +43,7 @@ output "web_reverse_proxy_lb_security_group_id" {
 output "web_hosprec_sg_id" {
     value = aws_security_group.web_hosprec.id
 }
+
+output "web_bulkdownload_sg_id" {
+  value = var.environment == "live" ? aws_security_group.web_bulkdownload[0].id : null
+}
diff --git a/components/terraform/security-groups/vars.tf b/components/terraform/security-groups/vars.tf
@@ -1,7 +1,7 @@
 variable "vpc_id" {}
 variable "lb_security_group_id" {}
 variable "instance_cidr" {}
-
+variable "environment" {}
 variable "redis_sg_id" {}
 
 variable "tags" {}
diff --git a/components/terraform/security-groups/web-bulkdownload-sgs.tf b/components/terraform/security-groups/web-bulkdownload-sgs.tf
@@ -0,0 +1,52 @@
+# -----------------------------------------------------------------------------
+# application servers Bulkdownload
+# -----------------------------------------------------------------------------
+resource "aws_security_group" "web_bulkdownload" {
+    count = var.environment == "live" ? 1 : 0
+
+    name        = "web-bulkdownload"
+    description = "access to bulkdownload application"
+    vpc_id      = var.vpc_id
+
+    tags = merge(var.tags, {
+        Name = "web-bulkdownload"
+    })
+}
+
+resource "aws_security_group_rule" "web_bulkdownload_http_ingress" {
+    count = var.environment == "live" ? 1 : 0
+
+    description              = "port 80 traffic from LB"
+    from_port                = 80
+    protocol                 = "tcp"
+    security_group_id        = aws_security_group.web_bulkdownload[0].id
+    source_security_group_id = var.lb_security_group_id
+    to_port                  = 80
+    type                     = "ingress"
+}
+
+resource "aws_security_group_rule" "web_bulkdownload_response_ingress" {
+    count = var.environment == "live" ? 1 : 0
+
+    cidr_blocks       = var.instance_cidr
+    description       = "traffic from Client-VPN and load balancer"
+    from_port         = 1024
+    protocol          = "tcp"
+    security_group_id = aws_security_group.web_bulkdownload[0].id
+    to_port           = 65535
+    type              = "ingress"
+}
+
+resource "aws_security_group_rule" "web_bulkdownload_http_egress" {
+    count = var.environment == "live" ? 1 : 0
+
+    security_group_id = aws_security_group.web_bulkdownload[0].id
+    type              = "egress"
+    from_port         = 0
+    to_port           = 0
+    protocol          = "-1"
+
+    cidr_blocks = [
+        "0.0.0.0/0"
+    ]
+}
diff --git a/components/terraform/web-bulkdownload.tf b/components/terraform/web-bulkdownload.tf
diff --git a/components/terraform/web-bulkdownload/scripts/startup.sh b/components/terraform/web-bulkdownload/scripts/startup.sh

Original file line number	Diff line number	Diff line change
`@@ -253,6 +253,7 @@ data "aws_ami" "web_hosprec_ami" {`
`253`	`253`	`}`
`254`	`254`
`255`	`255`	`data "aws_ami" "web_bulkdownload_ami" {`
	`256`	`+ count = var.environment == "live" ? 1 : 0`
`256`	`257`	`most_recent = true`
`257`	`258`
`258`	`259`	`filter {`