Add cronjob to run org list sync

This PR runs a cronjob every tuesday at 1130 that makes a rake call that fetches all organisations and syncs them to update the orgs list. We are currently running this as a manual job - the commit runs it automatically.

Author: alice-carr

infra/deployments/forms/forms-admin/main.tf

@@ -25,6 +25,7 @@ module "forms_admin" {
   analytics_enabled                  = var.forms_admin_settings.analytics_enabled
   act_as_user_enabled                = var.forms_admin_settings.act_as_user_enabled
   enable_mailchimp_sync              = var.forms_admin_settings.synchronize_to_mailchimp
+  enable_organisations_sync          = var.forms_admin_settings.synchronize_orgs_from_govuk
   deploy_account_id                  = var.deploy_account_id
   describe_none_of_the_above_enabled = var.forms_admin_settings.describe_none_of_the_above_enabled
   vpc_id                             = data.terraform_remote_state.forms_environment.outputs.vpc_id

infra/deployments/forms/inputs.tf

@@ -141,6 +141,7 @@ variable "forms_admin_settings" {
     act_as_user_enabled                = bool
     govuk_app_domain                   = string
     synchronize_to_mailchimp           = bool
+    synchronize_orgs_from_govuk        = bool
     describe_none_of_the_above_enabled = bool
   })
   nullable = false

infra/deployments/forms/pipelines/buildspecs/update-orgs-sync-task/update-orgs-sync-task.yml

@@ -0,0 +1,31 @@
+version: 0.2
+phases:
+  pre_build:
+    commands:
+      # Check Image URI is not the default pipeline value
+      - |
+        if [[ "${IMAGE_URI}" = "MUST_BE_SET" ]]; then
+          echo "The IMAGE_URI has not been set by the caller. The value of IMAGE_URI is \"${IMAGE_URI}\""
+          exit 1
+        fi
+
+      # Get task definition with tags
+      - echo "Existing orgs-sync task definition name is ${TASK_DEFINITION_NAME}"
+      - ECS_TASK_DEFINITION=$(aws ecs describe-task-definition --task-definition "${TASK_DEFINITION_NAME}" --include TAGS)
+
+      # Extract tags to preserve them
+      - |
+        TAGS=$(echo "${ECS_TASK_DEFINITION}" | jq -c '.tags // []')
+        echo "Preserving tags: ${TAGS}"
+
+      # Delete any reference to the old image from the task definition and add the new image uri
+      - |
+        NEW_ECS_TASK_DEFINITION=$(echo "${ECS_TASK_DEFINITION}" | jq --arg "INPUT_IMAGE_URI" "${IMAGE_URI}" '.taskDefinition | .containerDefinitions[0].image = $INPUT_IMAGE_URI | del(.taskDefinitionArn) | del(.revision) | del(.status) | del(.requiresAttributes) | del(.compatibilities) |  del(.registeredAt)  | del(.registeredBy)')
+        echo "Replaced image uri with ${IMAGE_URI}"
+
+  build:
+    commands:
+      # Register a new task definition with tags
+      - |
+        NEW_ECS_TASK_DEFINITION_ARN=$(aws ecs register-task-definition --cli-input-json "$NEW_ECS_TASK_DEFINITION" --tags "$TAGS" | jq -r '.taskDefinition.taskDefinitionArn' )
+        echo "New orgs-sync task definition ARN: ${NEW_ECS_TASK_DEFINITION_ARN}"

infra/deployments/forms/pipelines/deploy-forms-admin-container.tf

@@ -251,6 +251,35 @@ resource "aws_codepipeline" "deploy_admin_container" {
       }
     }
 
+    dynamic "action" {
+      for_each = var.forms_admin_settings.synchronize_orgs_from_govuk ? [1] : []
+      content {
+        name            = "update-orgs-sync-task-definition"
+        category        = "Build"
+        owner           = "AWS"
+        provider        = "CodeBuild"
+        version         = "1"
+        run_order       = 2
+        input_artifacts = ["buildspec_source"]
+        # AWS requires an input artifact; using buildspec_source as a relevant default.
+        configuration = {
+          ProjectName = module.update_orgs_sync_task_definition[0].name
+          EnvironmentVariables = jsonencode([
+            {
+              name  = "TASK_DEFINITION_NAME"
+              value = "${var.environment_name}_forms-admin_organisations_sync"
+              type  = "PLAINTEXT"
+            },
+            {
+              name  = "IMAGE_URI"
+              value = "#{variables.container_image_uri}"
+              type  = "PLAINTEXT"
+            }
+          ])
+        }
+      }
+    }
+
     # It isn't possible to conditionally skip or disable an action in CodePipeline
     # but we need to be able to do so because we can't run the end-to-end tests in the user-research
     # environment. We don't want to make the end-to-end tests module responsible for skipping itself
@@ -329,6 +358,19 @@ module "update_mailchimp_sync_task_definition" {
   codebuild_service_role_arn = data.aws_iam_role.deployer_role.arn
 }
 
+module "update_orgs_sync_task_definition" {
+  count = var.forms_admin_settings.synchronize_orgs_from_govuk ? 1 : 0
+
+  source                     = "../../../modules/code-build-build"
+  project_name               = "update_orgs_sync_task_definition_${var.environment_name}"
+  project_description        = "Update orgs-sync task definition with new container image"
+  environment                = var.environment_name
+  artifact_store_arn         = module.artifact_bucket.arn
+  buildspec                  = file("${path.root}/buildspecs/update-orgs-sync-task/update-orgs-sync-task.yml")
+  log_group_name             = "codebuild/update_orgs_sync_task_definition_${var.environment_name}"
+  codebuild_service_role_arn = data.aws_iam_role.deployer_role.arn
+}
+
 module "generate_forms_admin_container_image_defs" {
   source              = "../../../modules/code-build-build"
   project_name        = "generate_forms_admin_container_image_defs_${var.environment_name}"

infra/deployments/forms/tfvars/dev.tfvars

@@ -79,6 +79,7 @@ forms_admin_settings = {
   analytics_enabled                  = true
   act_as_user_enabled                = true
   govuk_app_domain                   = "integration.publishing.service.gov.uk"
+  synchronize_orgs_from_govuk        = false
   synchronize_to_mailchimp           = false
   describe_none_of_the_above_enabled = true
 }

infra/deployments/forms/tfvars/production.tfvars

@@ -136,6 +136,7 @@ forms_admin_settings = {
   act_as_user_enabled                = false
   govuk_app_domain                   = "publishing.service.gov.uk"
   synchronize_to_mailchimp           = true
+  synchronize_orgs_from_govuk        = true
   describe_none_of_the_above_enabled = false
 }
 forms_product_page_settings = {

infra/deployments/forms/tfvars/staging.tfvars

@@ -45,6 +45,7 @@ forms_admin_settings = {
   act_as_user_enabled                = true
   govuk_app_domain                   = "staging.publishing.service.gov.uk"
   synchronize_to_mailchimp           = false
+  synchronize_orgs_from_govuk        = false
   describe_none_of_the_above_enabled = false
 }
 forms_product_page_settings = {

infra/deployments/forms/tfvars/user-research.tfvars

@@ -43,6 +43,7 @@ forms_admin_settings = {
   act_as_user_enabled                = false
   govuk_app_domain                   = ""
   synchronize_to_mailchimp           = false
+  synchronize_orgs_from_govuk        = false
   describe_none_of_the_above_enabled = false
 }
 forms_product_page_settings = {

infra/modules/deployer-access/manage-ecs-service-policy.tf

@@ -104,7 +104,10 @@ data "aws_iam_policy_document" "ecs" {
       "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-admin-ecs-task-execution",
       "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-runner-ecs-task-execution",
       "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-product-page-ecs-task-execution",
-      "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-runner-queue-worker-ecs-task-execution"
+      "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-runner-queue-worker-ecs-task-execution",
+      "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-admin-ecs-cron-scheduler",           #TODO: remove after deployment
+      "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-admin-mailchimp-ecs-cron-scheduler", #TODO: remove after deployment
+      "arn:aws:iam::${var.account_id}:role/${var.environment_name}-forms-admin-orgs-ecs-cron-scheduler",      #TODO: remove after deployment
     ]
     effect = "Allow"
   }

infra/modules/forms-admin/mailchimp-sync.tf

@@ -21,7 +21,7 @@ locals {
   )
 }
 
-resource "aws_ecs_task_definition" "cron_job" {
+resource "aws_ecs_task_definition" "mailchimp_cron_job" {
   count = var.enable_mailchimp_sync ? 1 : 0
 
   family                = "${var.env_name}_forms-admin_mailchimp_sync"
@@ -46,27 +46,27 @@ resource "aws_ecs_task_definition" "cron_job" {
 ##
 # EventBridge
 ##
-resource "aws_cloudwatch_event_rule" "sync_cron_job" {
+resource "aws_cloudwatch_event_rule" "sync_mailchimp_cron_job" {
   count = var.enable_mailchimp_sync ? 1 : 0
 
-  name                = "${var.env_name}-forms-admin-sync-cron"
+  name                = "${var.env_name}-forms-admin-mailchimp-sync-cron"
   description         = "Trigger the forms-admin MailChimp synchronisation on a schedule"
   schedule_expression = "cron(30 10 * * ? *)" # 10:30AM daily. In office hours so that we can respond to failures
 }
 
-resource "aws_cloudwatch_event_target" "ecs_sync_job" {
+resource "aws_cloudwatch_event_target" "ecs_mailchimp_sync_job" {
   count = var.enable_mailchimp_sync ? 1 : 0
 
   arn      = var.ecs_cluster_arn
-  rule     = aws_cloudwatch_event_rule.sync_cron_job[0].name
-  role_arn = aws_iam_role.ecs_cron_scheduler[0].arn
+  rule     = aws_cloudwatch_event_rule.sync_mailchimp_cron_job[0].name
+  role_arn = aws_iam_role.ecs_mailchimp_cron_scheduler[0].arn
 
   ecs_target {
     # Construct ARN without revision number to always use the latest revision
     # Format: arn:aws:ecs:region:account:task-definition/family
     # This ensures the EventBridge rule always uses the latest revision
     # which is updated by the forms-admin deployment pipeline
-    task_definition_arn = "arn:aws:ecs:eu-west-2:${data.aws_caller_identity.current.account_id}:task-definition/${aws_ecs_task_definition.cron_job[0].family}"
+    task_definition_arn = "arn:aws:ecs:eu-west-2:${data.aws_caller_identity.current.account_id}:task-definition/${aws_ecs_task_definition.mailchimp_cron_job[0].family}"
     launch_type         = "FARGATE"
     platform_version    = "1.4.0"
 
@@ -83,8 +83,8 @@ resource "aws_cloudwatch_event_target" "ecs_sync_job" {
 }
 
 ## Monitor for failure
-resource "aws_cloudwatch_event_rule" "sync_cron_job_failed" {
-  name        = "${var.env_name}-forms-admin-sync-failed"
+resource "aws_cloudwatch_event_rule" "sync_mailchimp_cron_job_failed" {
+  name        = "${var.env_name}-forms-admin-mailchimp-sync-failed"
   description = "Trigger when the MailChimp sync job has exited with a non-zero exit code"
 
   event_pattern = jsonencode({
@@ -106,8 +106,8 @@ resource "aws_cloudwatch_event_rule" "sync_cron_job_failed" {
   })
 }
 
-resource "aws_cloudwatch_event_target" "sync_cron_job_alert_message" {
-  rule = aws_cloudwatch_event_rule.sync_cron_job_failed.name
+resource "aws_cloudwatch_event_target" "sync_mailchimp_cron_job_alert_message" {
+  rule = aws_cloudwatch_event_rule.sync_mailchimp_cron_job_failed.name
 
   # defined in 'environment' module. Sends alarms/errors via ZenDesk
   arn = var.zendesk_sns_topic_arn
@@ -134,10 +134,10 @@ resource "aws_cloudwatch_event_target" "sync_cron_job_alert_message" {
 ##
 # IAM
 ##
-resource "aws_iam_role" "ecs_cron_scheduler" {
+resource "aws_iam_role" "ecs_mailchimp_cron_scheduler" {
   count = var.enable_mailchimp_sync ? 1 : 0
 
-  name = "${var.env_name}-forms-admin-ecs-cron-scheduler"
+  name = "${var.env_name}-forms-admin-mailchimp-ecs-cron-scheduler"
 
   assume_role_policy = jsonencode({
     Version = "2012-10-17"
@@ -153,9 +153,19 @@ resource "aws_iam_role" "ecs_cron_scheduler" {
   })
 }
 
-resource "aws_iam_role_policy_attachment" "ecs_events_policy" {
+resource "aws_iam_role_policy_attachment" "ecs_mailchimp_events_policy" {
   count = var.enable_mailchimp_sync ? 1 : 0
 
   policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceEventsRole"
-  role       = aws_iam_role.ecs_cron_scheduler[0].name
+  role       = aws_iam_role.ecs_mailchimp_cron_scheduler[0].name
+}
+
+moved {
+  from = aws_cloudwatch_event_rule.sync_cron_job_failed
+  to   = aws_cloudwatch_event_rule.sync_mailchimp_cron_job_failed
+}
+
+moved {
+  from = aws_cloudwatch_event_target.sync_cron_job_alert_message
+  to   = aws_cloudwatch_event_target.sync_mailchimp_cron_job_alert_message
 }