Skip to content

Commit ce36f44

Browse files
authored
Merge pull request #1646 from alphagov/add-org-sync-cron-job
Add cronjob to run org list sync
2 parents d7df588 + 27b50cb commit ce36f44

11 files changed

Lines changed: 269 additions & 15 deletions

File tree

infra/deployments/forms/forms-admin/main.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ module "forms_admin" {
2525
analytics_enabled = var.forms_admin_settings.analytics_enabled
2626
act_as_user_enabled = var.forms_admin_settings.act_as_user_enabled
2727
enable_mailchimp_sync = var.forms_admin_settings.synchronize_to_mailchimp
28+
enable_organisations_sync = var.forms_admin_settings.synchronize_orgs_from_govuk
2829
deploy_account_id = var.deploy_account_id
2930
describe_none_of_the_above_enabled = var.forms_admin_settings.describe_none_of_the_above_enabled
3031
vpc_id = data.terraform_remote_state.forms_environment.outputs.vpc_id

infra/deployments/forms/inputs.tf

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ variable "forms_admin_settings" {
141141
act_as_user_enabled = bool
142142
govuk_app_domain = string
143143
synchronize_to_mailchimp = bool
144+
synchronize_orgs_from_govuk = bool
144145
describe_none_of_the_above_enabled = bool
145146
})
146147
nullable = false
Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
version: 0.2
2+
phases:
3+
pre_build:
4+
commands:
5+
# Check Image URI is not the default pipeline value
6+
- |
7+
if [[ "${IMAGE_URI}" = "MUST_BE_SET" ]]; then
8+
echo "The IMAGE_URI has not been set by the caller. The value of IMAGE_URI is \"${IMAGE_URI}\""
9+
exit 1
10+
fi
11+
12+
# Get task definition with tags
13+
- echo "Existing orgs-sync task definition name is ${TASK_DEFINITION_NAME}"
14+
- ECS_TASK_DEFINITION=$(aws ecs describe-task-definition --task-definition "${TASK_DEFINITION_NAME}" --include TAGS)
15+
16+
# Extract tags to preserve them
17+
- |
18+
TAGS=$(echo "${ECS_TASK_DEFINITION}" | jq -c '.tags // []')
19+
echo "Preserving tags: ${TAGS}"
20+
21+
# Delete any reference to the old image from the task definition and add the new image uri
22+
- |
23+
NEW_ECS_TASK_DEFINITION=$(echo "${ECS_TASK_DEFINITION}" | jq --arg "INPUT_IMAGE_URI" "${IMAGE_URI}" '.taskDefinition | .containerDefinitions[0].image = $INPUT_IMAGE_URI | del(.taskDefinitionArn) | del(.revision) | del(.status) | del(.requiresAttributes) | del(.compatibilities) | del(.registeredAt) | del(.registeredBy)')
24+
echo "Replaced image uri with ${IMAGE_URI}"
25+
26+
build:
27+
commands:
28+
# Register a new task definition with tags
29+
- |
30+
NEW_ECS_TASK_DEFINITION_ARN=$(aws ecs register-task-definition --cli-input-json "$NEW_ECS_TASK_DEFINITION" --tags "$TAGS" | jq -r '.taskDefinition.taskDefinitionArn' )
31+
echo "New orgs-sync task definition ARN: ${NEW_ECS_TASK_DEFINITION_ARN}"

infra/deployments/forms/pipelines/deploy-forms-admin-container.tf

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -251,6 +251,35 @@ resource "aws_codepipeline" "deploy_admin_container" {
251251
}
252252
}
253253

254+
dynamic "action" {
255+
for_each = var.forms_admin_settings.synchronize_orgs_from_govuk ? [1] : []
256+
content {
257+
name = "update-orgs-sync-task-definition"
258+
category = "Build"
259+
owner = "AWS"
260+
provider = "CodeBuild"
261+
version = "1"
262+
run_order = 2
263+
input_artifacts = ["buildspec_source"]
264+
# AWS requires an input artifact; using buildspec_source as a relevant default.
265+
configuration = {
266+
ProjectName = module.update_orgs_sync_task_definition[0].name
267+
EnvironmentVariables = jsonencode([
268+
{
269+
name = "TASK_DEFINITION_NAME"
270+
value = "${var.environment_name}_forms-admin_organisations_sync"
271+
type = "PLAINTEXT"
272+
},
273+
{
274+
name = "IMAGE_URI"
275+
value = "#{variables.container_image_uri}"
276+
type = "PLAINTEXT"
277+
}
278+
])
279+
}
280+
}
281+
}
282+
254283
# It isn't possible to conditionally skip or disable an action in CodePipeline
255284
# but we need to be able to do so because we can't run the end-to-end tests in the user-research
256285
# environment. We don't want to make the end-to-end tests module responsible for skipping itself
@@ -329,6 +358,19 @@ module "update_mailchimp_sync_task_definition" {
329358
codebuild_service_role_arn = data.aws_iam_role.deployer_role.arn
330359
}
331360

361+
module "update_orgs_sync_task_definition" {
362+
count = var.forms_admin_settings.synchronize_orgs_from_govuk ? 1 : 0
363+
364+
source = "../../../modules/code-build-build"
365+
project_name = "update_orgs_sync_task_definition_${var.environment_name}"
366+
project_description = "Update orgs-sync task definition with new container image"
367+
environment = var.environment_name
368+
artifact_store_arn = module.artifact_bucket.arn
369+
buildspec = file("${path.root}/buildspecs/update-orgs-sync-task/update-orgs-sync-task.yml")
370+
log_group_name = "codebuild/update_orgs_sync_task_definition_${var.environment_name}"
371+
codebuild_service_role_arn = data.aws_iam_role.deployer_role.arn
372+
}
373+
332374
module "generate_forms_admin_container_image_defs" {
333375
source = "../../../modules/code-build-build"
334376
project_name = "generate_forms_admin_container_image_defs_${var.environment_name}"

infra/deployments/forms/tfvars/dev.tfvars

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@ forms_admin_settings = {
7979
analytics_enabled = true
8080
act_as_user_enabled = true
8181
govuk_app_domain = "integration.publishing.service.gov.uk"
82+
synchronize_orgs_from_govuk = false
8283
synchronize_to_mailchimp = false
8384
describe_none_of_the_above_enabled = true
8485
}

infra/deployments/forms/tfvars/production.tfvars

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,7 @@ forms_admin_settings = {
136136
act_as_user_enabled = false
137137
govuk_app_domain = "publishing.service.gov.uk"
138138
synchronize_to_mailchimp = true
139+
synchronize_orgs_from_govuk = true
139140
describe_none_of_the_above_enabled = false
140141
}
141142
forms_product_page_settings = {

infra/deployments/forms/tfvars/staging.tfvars

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ forms_admin_settings = {
4444
analytics_enabled = true
4545
act_as_user_enabled = true
4646
govuk_app_domain = "staging.publishing.service.gov.uk"
47+
synchronize_orgs_from_govuk = false
4748
synchronize_to_mailchimp = false
4849
describe_none_of_the_above_enabled = false
4950
}

infra/deployments/forms/tfvars/user-research.tfvars

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@ forms_admin_settings = {
4242
analytics_enabled = false
4343
act_as_user_enabled = false
4444
govuk_app_domain = ""
45+
synchronize_orgs_from_govuk = false
4546
synchronize_to_mailchimp = false
4647
describe_none_of_the_above_enabled = false
4748
}

infra/modules/forms-admin/mailchimp-sync.tf

Lines changed: 25 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ locals {
2121
)
2222
}
2323

24-
resource "aws_ecs_task_definition" "cron_job" {
24+
resource "aws_ecs_task_definition" "mailchimp_cron_job" {
2525
count = var.enable_mailchimp_sync ? 1 : 0
2626

2727
family = "${var.env_name}_forms-admin_mailchimp_sync"
@@ -46,27 +46,27 @@ resource "aws_ecs_task_definition" "cron_job" {
4646
##
4747
# EventBridge
4848
##
49-
resource "aws_cloudwatch_event_rule" "sync_cron_job" {
49+
resource "aws_cloudwatch_event_rule" "sync_mailchimp_cron_job" {
5050
count = var.enable_mailchimp_sync ? 1 : 0
5151

52-
name = "${var.env_name}-forms-admin-sync-cron"
52+
name = "${var.env_name}-forms-admin-mailchimp-sync-cron"
5353
description = "Trigger the forms-admin MailChimp synchronisation on a schedule"
5454
schedule_expression = "cron(30 10 * * ? *)" # 10:30AM daily. In office hours so that we can respond to failures
5555
}
5656

57-
resource "aws_cloudwatch_event_target" "ecs_sync_job" {
57+
resource "aws_cloudwatch_event_target" "ecs_mailchimp_sync_job" {
5858
count = var.enable_mailchimp_sync ? 1 : 0
5959

6060
arn = var.ecs_cluster_arn
61-
rule = aws_cloudwatch_event_rule.sync_cron_job[0].name
62-
role_arn = aws_iam_role.ecs_cron_scheduler[0].arn
61+
rule = aws_cloudwatch_event_rule.sync_mailchimp_cron_job[0].name
62+
role_arn = aws_iam_role.ecs_mailchimp_cron_scheduler[0].arn
6363

6464
ecs_target {
6565
# Construct ARN without revision number to always use the latest revision
6666
# Format: arn:aws:ecs:region:account:task-definition/family
6767
# This ensures the EventBridge rule always uses the latest revision
6868
# which is updated by the forms-admin deployment pipeline
69-
task_definition_arn = "arn:aws:ecs:eu-west-2:${data.aws_caller_identity.current.account_id}:task-definition/${aws_ecs_task_definition.cron_job[0].family}"
69+
task_definition_arn = "arn:aws:ecs:eu-west-2:${data.aws_caller_identity.current.account_id}:task-definition/${aws_ecs_task_definition.mailchimp_cron_job[0].family}"
7070
launch_type = "FARGATE"
7171
platform_version = "1.4.0"
7272

@@ -83,8 +83,8 @@ resource "aws_cloudwatch_event_target" "ecs_sync_job" {
8383
}
8484

8585
## Monitor for failure
86-
resource "aws_cloudwatch_event_rule" "sync_cron_job_failed" {
87-
name = "${var.env_name}-forms-admin-sync-failed"
86+
resource "aws_cloudwatch_event_rule" "sync_mailchimp_cron_job_failed" {
87+
name = "${var.env_name}-forms-admin-mailchimp-sync-failed"
8888
description = "Trigger when the MailChimp sync job has exited with a non-zero exit code"
8989

9090
event_pattern = jsonencode({
@@ -106,8 +106,8 @@ resource "aws_cloudwatch_event_rule" "sync_cron_job_failed" {
106106
})
107107
}
108108

109-
resource "aws_cloudwatch_event_target" "sync_cron_job_alert_message" {
110-
rule = aws_cloudwatch_event_rule.sync_cron_job_failed.name
109+
resource "aws_cloudwatch_event_target" "sync_mailchimp_cron_job_alert_message" {
110+
rule = aws_cloudwatch_event_rule.sync_mailchimp_cron_job_failed.name
111111

112112
# defined in 'environment' module. Sends alarms/errors via ZenDesk
113113
arn = var.zendesk_sns_topic_arn
@@ -134,10 +134,10 @@ resource "aws_cloudwatch_event_target" "sync_cron_job_alert_message" {
134134
##
135135
# IAM
136136
##
137-
resource "aws_iam_role" "ecs_cron_scheduler" {
137+
resource "aws_iam_role" "ecs_mailchimp_cron_scheduler" {
138138
count = var.enable_mailchimp_sync ? 1 : 0
139139

140-
name = "${var.env_name}-forms-admin-ecs-cron-scheduler"
140+
name = "${var.env_name}-forms-admin-mailchimp-ecs-cron-scheduler"
141141

142142
assume_role_policy = jsonencode({
143143
Version = "2012-10-17"
@@ -153,9 +153,19 @@ resource "aws_iam_role" "ecs_cron_scheduler" {
153153
})
154154
}
155155

156-
resource "aws_iam_role_policy_attachment" "ecs_events_policy" {
156+
resource "aws_iam_role_policy_attachment" "ecs_mailchimp_events_policy" {
157157
count = var.enable_mailchimp_sync ? 1 : 0
158158

159159
policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceEventsRole"
160-
role = aws_iam_role.ecs_cron_scheduler[0].name
160+
role = aws_iam_role.ecs_mailchimp_cron_scheduler[0].name
161+
}
162+
163+
moved {
164+
from = aws_cloudwatch_event_rule.sync_cron_job_failed
165+
to = aws_cloudwatch_event_rule.sync_mailchimp_cron_job_failed
166+
}
167+
168+
moved {
169+
from = aws_cloudwatch_event_target.sync_cron_job_alert_message
170+
to = aws_cloudwatch_event_target.sync_mailchimp_cron_job_alert_message
161171
}
Lines changed: 159 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,159 @@
1+
##
2+
# ECS
3+
##
4+
locals {
5+
# Take the exported task container definition
6+
# and override some parts of it, so that it doesn't fall out of sync
7+
organisations_sync_container_definitions = merge(
8+
module.ecs_service.task_container_definition,
9+
{
10+
name = "forms-admin_organisations_sync",
11+
command = ["rake", "organisations:fetch"]
12+
logConfiguration = {
13+
logDriver = "awslogs",
14+
options = {
15+
awslogs-group = module.ecs_service.application_log_group_name,
16+
awslogs-region = "eu-west-2",
17+
awslogs-stream-prefix = "forms-admin-${var.env_name}-organisations-sync"
18+
}
19+
},
20+
}
21+
)
22+
}
23+
24+
resource "aws_ecs_task_definition" "orgs_cron_job" {
25+
count = var.enable_organisations_sync ? 1 : 0
26+
27+
family = "${var.env_name}_forms-admin_organisations_sync"
28+
container_definitions = jsonencode([local.organisations_sync_container_definitions])
29+
30+
execution_role_arn = module.ecs_service.task_definition.execution_role_arn
31+
task_role_arn = module.ecs_service.task_definition.task_role_arn
32+
requires_compatibilities = module.ecs_service.task_definition.requires_compatibilities
33+
cpu = module.ecs_service.task_definition.cpu
34+
memory = module.ecs_service.task_definition.memory
35+
network_mode = "awsvpc"
36+
track_latest = true
37+
38+
runtime_platform {
39+
operating_system_family = "LINUX"
40+
cpu_architecture = "ARM64"
41+
}
42+
}
43+
44+
##
45+
# EventBridge
46+
##
47+
resource "aws_cloudwatch_event_rule" "sync_orgs_cron_job" {
48+
count = var.enable_organisations_sync ? 1 : 0
49+
50+
name = "${var.env_name}-forms-admin-orgs-sync-cron"
51+
description = "Trigger the forms-admin organisations synchronisation on a schedule"
52+
schedule_expression = "cron(30 11 * * 2 *)" # 11:30AM every Tuesday. In office hours so that we can respond to failures
53+
}
54+
55+
resource "aws_cloudwatch_event_target" "ecs_org_sync_job" {
56+
count = var.enable_organisations_sync ? 1 : 0
57+
58+
arn = var.ecs_cluster_arn
59+
rule = aws_cloudwatch_event_rule.sync_orgs_cron_job[0].name
60+
role_arn = aws_iam_role.ecs_orgs_cron_scheduler[0].arn
61+
62+
ecs_target {
63+
task_definition_arn = "arn:aws:ecs:eu-west-2:${data.aws_caller_identity.current.account_id}:task-definition/${aws_ecs_task_definition.orgs_cron_job[0].family}"
64+
launch_type = "FARGATE"
65+
platform_version = "1.4.0"
66+
67+
network_configuration {
68+
assign_public_ip = false
69+
security_groups = module.ecs_service.service.network_configuration[0].security_groups
70+
subnets = module.ecs_service.service.network_configuration[0].subnets
71+
}
72+
}
73+
74+
dead_letter_config {
75+
arn = var.eventbridge_dead_letter_queue_arn
76+
}
77+
}
78+
79+
## Monitor for failure
80+
resource "aws_cloudwatch_event_rule" "sync_orgs_cron_job_failed" {
81+
count = var.enable_organisations_sync ? 1 : 0
82+
83+
name = "${var.env_name}-forms-admin-org-sync-failed"
84+
description = "Trigger when the organisations sync job has exited with a non-zero exit code"
85+
86+
event_pattern = jsonencode({
87+
source = ["aws.ecs"]
88+
detail-type = ["ECS Task State Change"]
89+
resources = [
90+
{
91+
wildcard : "arn:aws:ecs:eu-west-2:${data.aws_caller_identity.current.account_id}:task/*"
92+
}
93+
]
94+
95+
detail = {
96+
lastStatus = ["STOPPED"]
97+
containers = {
98+
name = [local.organisations_sync_container_definitions.name]
99+
exitCode = [{ "anything-but" : [0] }]
100+
}
101+
}
102+
})
103+
}
104+
105+
resource "aws_cloudwatch_event_target" "sync_orgs_cron_job_alert_message" {
106+
count = var.enable_organisations_sync ? 1 : 0
107+
108+
rule = aws_cloudwatch_event_rule.sync_orgs_cron_job_failed[0].name
109+
110+
# defined in 'environment' module. Sends alarms/errors via ZenDesk
111+
arn = var.zendesk_sns_topic_arn
112+
113+
input_transformer {
114+
input_template = <<EOF
115+
{
116+
"title": "WARNING: Synchronising organisations from GOV.UK has failed.",
117+
"description": "GOV.UK Forms has a scheduled ECS task to sync our organisations from GOV.UK. When this task fails an email is sent to Zendesk.",
118+
"next-steps": {
119+
"1": "Navigate to Splunk: https://gds.splunkcloud.com/en-GB/app/gds-543-forms/search.",
120+
"2": "Search for index=gds_dsp_production_forms log_stream=forms-admin-production-organisations-sync/forms-admin_organisations_sync/*. Use the 'Today' date-time preset to find today's logs.",
121+
"3": "Review logs for errors."
122+
}
123+
}
124+
EOF
125+
}
126+
127+
dead_letter_config {
128+
arn = var.eventbridge_dead_letter_queue_arn
129+
}
130+
}
131+
132+
##
133+
# IAM
134+
##
135+
resource "aws_iam_role" "ecs_orgs_cron_scheduler" {
136+
count = var.enable_organisations_sync ? 1 : 0
137+
138+
name = "${var.env_name}-forms-admin-orgs-ecs-cron-scheduler"
139+
140+
assume_role_policy = jsonencode({
141+
Version = "2012-10-17"
142+
Statement = [
143+
{
144+
Action = "sts:AssumeRole"
145+
Effect = "Allow"
146+
Principal = {
147+
Service = "events.amazonaws.com"
148+
}
149+
}
150+
]
151+
})
152+
}
153+
154+
resource "aws_iam_role_policy_attachment" "ecs_orgs_events_policy" {
155+
count = var.enable_organisations_sync ? 1 : 0
156+
157+
policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonEC2ContainerServiceEventsRole"
158+
role = aws_iam_role.ecs_orgs_cron_scheduler[0].name
159+
}

0 commit comments

Comments
 (0)