Skip to content

Commit a7a4c08

Browse files
dturner5234dturner5234
authored
Merge pull request #135 from DSACMS/wbprice/dagster-ecs-service-attempt-2
Wbprice/dagster ecs service attempt 2
2 parents 5182039 + 0a2b8ac commit a7a4c08

File tree

19 files changed

+536
-85
lines changed

19 files changed

+536
-85
lines changed

‎infrastructure/envs/dev/main.tf‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ data "aws_vpc" "default" {
3131
}
3232
}
3333

34+
module "repositories" {
35+
source = "../../modules/repositories"
36+
37+
account_name = local.account_name
38+
}
39+
3440
module "networking" {
3541
source = "../../modules/networking"
3642

@@ -71,6 +77,7 @@ module "etl-db" {
7177
allocated_storage = 100
7278
publicly_accessible = false
7379
username = "npd_etl"
80+
db_name = "npd_etl"
7481
vpc_security_group_ids = [module.networking.db_security_group_id]
7582
db_subnet_group_name = module.networking.db_subnet_group_name
7683
backup_retention_period = 7 # Remove automated snapshots after 7 days
@@ -122,7 +129,22 @@ module "fhir-api" {
122129
module "etl" {
123130
source = "../../modules/etl"
124131

125-
account_name = local.account_name
132+
account_name = local.account_name
133+
dagster_image = var.dagster_image
134+
ecs_cluster_id = module.ecs.cluster_id
135+
db = {
136+
db_instance_master_user_secret_arn = module.etl-db.db_instance_master_user_secret_arn
137+
db_instance_address = module.etl-db.db_instance_address
138+
db_instance_port = module.etl-db.db_instance_port
139+
db_instance_name = module.etl-db.db_instance_name
140+
}
141+
networking = {
142+
etl_subnet_ids = module.networking.etl_subnet_ids
143+
etl_security_group_id = module.networking.etl_security_group_id
144+
etl_alb_security_group_id = module.networking.etl_alb_security_group_id
145+
public_subnet_ids = module.networking.public_subnet_ids
146+
vpc_id = module.networking.vpc_id
147+
}
126148
}
127149

128150
# Frontend Module

‎infrastructure/envs/dev/outputs.tf‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,18 @@ output "etl_db_instance_endpoint" {
1010
value = module.etl-db.db_instance_endpoint
1111
}
1212

13-
output "api_ecr_repository_name" {
14-
value = module.fhir-api.api_ecr_repository_name
13+
output "fhir_api_repository_name" {
14+
value = module.repositories.fhir_api_repository_name
1515
}
1616

17-
output "api_migrations_ecr_repository_name" {
18-
value = module.fhir-api.api_migrations_ecr_repository_name
17+
output "fhir_api_migrations_repository_name" {
18+
value = module.repositories.fhir_api_migrations_repository_name
19+
}
20+
21+
output "dagster_repository_name" {
22+
value = module.repositories.dagster_repository_name
23+
}
24+
25+
output "dagster_ui_alb_dns_name" {
26+
value = module.etl.dagster_ui_alb_dns_name
1927
}

‎infrastructure/envs/dev/variables.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,3 +8,4 @@ variable "tier" {
88

99
variable "migration_image" { default = "575012135727.dkr.ecr.us-east-1.amazonaws.com/npd-east-dev-fhir-api-migrations:latest" }
1010
variable "fhir_api_image" { default = "575012135727.dkr.ecr.us-east-1.amazonaws.com/npd-east-dev-fhir-api:latest" }
11+
variable "dagster_image" { default = "575012135727.dkr.ecr.us-east-1.amazonaws.com/npd-east-dev-dagster:latest" }

‎infrastructure/envs/prod/main.tf‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,12 @@ data "aws_vpc" "default" {
3131
}
3232
}
3333

34+
module "repositories" {
35+
source = "../../modules/repositories"
36+
37+
account_name = local.account_name
38+
}
39+
3440
module "networking" {
3541
source = "../../modules/networking"
3642

@@ -122,7 +128,22 @@ module "fhir-api" {
122128
module "etl" {
123129
source = "../../modules/etl"
124130

125-
account_name = local.account_name
131+
account_name = local.account_name
132+
dagster_image = var.dagster_image
133+
ecs_cluster_id = module.ecs.cluster_id
134+
db = {
135+
db_instance_master_user_secret_arn = module.etl-db.db_instance_master_user_secret_arn
136+
db_instance_address = module.etl-db.db_instance_address
137+
db_instance_port = module.etl-db.db_instance_port
138+
db_instance_name = module.etl-db.db_instance_name
139+
}
140+
networking = {
141+
etl_subnet_ids = module.networking.etl_subnet_ids
142+
etl_security_group_id = module.networking.etl_security_group_id
143+
etl_alb_security_group_id = module.networking.etl_alb_security_group_id
144+
public_subnet_ids = module.networking.public_subnet_ids
145+
vpc_id = module.networking.vpc_id
146+
}
126147
}
127148

128149
# Frontend Module

‎infrastructure/envs/prod/outputs.tf‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,18 @@ output "etl_db_instance_endpoint" {
1010
value = module.etl-db.db_instance_endpoint
1111
}
1212

13-
output "api_ecr_repository_name" {
14-
value = module.fhir-api.api_ecr_repository_name
13+
output "fhir_api_repository_name" {
14+
value = module.repositories.fhir_api_repository_name
1515
}
1616

17-
output "api_migrations_ecr_repository_name" {
18-
value = module.fhir-api.api_migrations_ecr_repository_name
17+
output "fhir_api_migrations_repository_name" {
18+
value = module.repositories.fhir_api_migrations_repository_name
19+
}
20+
21+
output "dagster_repository_name" {
22+
value = module.repositories.dagster_repository_name
23+
}
24+
25+
output "dagster_ui_alb_dns_name" {
26+
value = module.etl.dagster_ui_alb_dns_name
1927
}

‎infrastructure/envs/prod/variables.tf‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,4 +8,5 @@ variable "tier" {
88

99
variable "migration_image" { default = "596240962403.dkr.ecr.us-east-1.amazonaws.com/npd-east-prod-fhir-api-migrations:latest" }
1010
variable "fhir_api_image" { default = "596240962403.dkr.ecr.us-east-1.amazonaws.com/npd-east-prod-fhir-api:latest" }
11+
variable "dagster_image" { default = "596240962403.dkr.ecr.us-east-1.amazonaws.com/npd-east-dev-dagster:latest" }
1112
variable "redirect_to_strategy_page" { default = true }

‎infrastructure/envs/sandbox/main.tf‎

Lines changed: 13 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ terraform {
99
}
1010

1111
backend "s3" {
12-
bucket = "npd-terraform"
13-
key = "terraform.tfstate"
14-
region = "us-gov-west-1"
12+
bucket = "npd-terraform"
13+
key = "terraform.tfstate"
14+
region = "us-gov-west-1"
1515
use_lockfile = true
1616
}
1717
}
@@ -107,7 +107,7 @@ resource "aws_iam_role_policy_attachment" "ecs_task_execution" {
107107
}
108108

109109
resource "aws_iam_policy" "ecs_task_can_access_database_secret" {
110-
name = "ecs-task-can-access-database-secret"
110+
name = "ecs-task-can-access-database-secret"
111111
description = "Allows ECS tasks to access the RDS secret from Secrets Manager"
112112
policy = jsonencode({
113113
Version = "2012-10-17"
@@ -125,7 +125,7 @@ resource "aws_iam_policy" "ecs_task_can_access_database_secret" {
125125
}
126126

127127
resource "aws_iam_role_policy_attachment" "ecs_task_can_access_database_secret_attachement" {
128-
role = aws_iam_role.ecs_task_execution.name
128+
role = aws_iam_role.ecs_task_execution.name
129129
policy_arn = aws_iam_policy.ecs_task_can_access_database_secret.arn
130130
}
131131

@@ -168,8 +168,8 @@ resource "aws_secretsmanager_secret" "django_secret" {
168168
}
169169

170170
resource "aws_secretsmanager_secret_version" "django_secret_version" {
171-
secret_id = aws_secretsmanager_secret.django_secret.id
172-
secret_string_wo = data.aws_secretsmanager_random_password.django_secret_value.random_password
171+
secret_id = aws_secretsmanager_secret.django_secret.id
172+
secret_string_wo = data.aws_secretsmanager_random_password.django_secret_value.random_password
173173
secret_string_wo_version = 1
174174
}
175175

@@ -192,10 +192,10 @@ resource "aws_ecs_task_definition" "app" {
192192
name = "${var.name}-migrations"
193193
image = var.migration_image
194194
essential = false
195-
command = [ "migrate" ]
195+
command = ["migrate"]
196196
environment = [
197197
{
198-
name = "FLYWAY_URL"
198+
name = "FLYWAY_URL"
199199
value = "jdbc:postgresql://${module.rds.db_instance_address}:${module.rds.db_instance_port}/${var.app_db_name}"
200200
}
201201
],
@@ -222,7 +222,7 @@ resource "aws_ecs_task_definition" "app" {
222222
name = var.name
223223
image = var.container_image
224224
essential = true
225-
environment = [
225+
environment = [
226226
{
227227
name = "NPD_DB_NAME"
228228
value = var.app_db_name
@@ -240,7 +240,7 @@ resource "aws_ecs_task_definition" "app" {
240240
value = "django.db.backends.postgresql"
241241
},
242242
{
243-
name = "DEBUG"
243+
name = "DEBUG"
244244
value = ""
245245
},
246246
{
@@ -256,7 +256,7 @@ resource "aws_ecs_task_definition" "app" {
256256
value = "ndh"
257257
},
258258
{
259-
name = "CACHE_LOCATION",
259+
name = "CACHE_LOCATION",
260260
value = ""
261261
}
262262
]
@@ -446,7 +446,7 @@ module "rds" {
446446
publicly_accessible = false
447447
vpc_security_group_ids = [aws_security_group.rds_sg.id]
448448
db_subnet_group_name = aws_db_subnet_group.db.name
449-
backup_retention_period = 7 # Remove automated snapshots after 7 days
449+
backup_retention_period = 7 # Remove automated snapshots after 7 days
450450
backup_window = "03:00-04:00" # 11PM EST
451451
}
452452

‎infrastructure/envs/sandbox/terraform.tfvars‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
name = "ndh"
1+
name = "ndh"
22
db_name = "ndh"

‎infrastructure/modules/aws-glue-poc/aws-glue-poc.tf‎

Lines changed: 29 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -4,35 +4,35 @@ main.tf for the time being.
44
*/
55

66
resource "aws_security_group" "glue_sg" {
7-
name = "glue-sg"
7+
name = "glue-sg"
88
description = "Common security group for Glue jobs"
9-
vpc_id = data.aws_vpc.default.id
9+
vpc_id = data.aws_vpc.default.id
1010
}
1111

1212
resource "aws_vpc_security_group_ingress_rule" "glue_sg_allow_connections_from_self" {
13-
security_group_id = aws_security_group.glue_sg.id
14-
ip_protocol = "-1"
13+
security_group_id = aws_security_group.glue_sg.id
14+
ip_protocol = "-1"
1515
referenced_security_group_id = aws_security_group.glue_sg.id
1616
}
1717

1818
resource "aws_vpc_security_group_egress_rule" "glue_gs_allow_outbound_connections" {
1919
security_group_id = aws_security_group.glue_sg.id
20-
ip_protocol = "-1"
21-
cidr_ipv4 = "0.0.0.0/0"
20+
ip_protocol = "-1"
21+
cidr_ipv4 = "0.0.0.0/0"
2222
}
2323

2424
resource "aws_s3_bucket" "glue_scripts" {
25-
bucket = "${var.name}-glue-scripts-bucket"
25+
bucket = "${var.name}-glue-scripts-bucket"
2626
force_destroy = true
2727
}
2828

2929
resource "aws_s3_bucket" "aws_glue_input_bucket" {
30-
bucket = "${var.name}-glue-s3-input"
30+
bucket = "${var.name}-glue-s3-input"
3131
force_destroy = true
3232
}
3333

3434
resource "aws_s3_bucket" "aws_glue_output_bucket" {
35-
bucket = "${var.name}-glue-s3-output"
35+
bucket = "${var.name}-glue-s3-output"
3636
force_destroy = true
3737
}
3838

@@ -41,7 +41,7 @@ resource "aws_glue_catalog_database" "aws_glue_catalog" {
4141
}
4242

4343
resource "aws_glue_crawler" "aws_glue_crawler" {
44-
database_name = aws_glue_catalog_database.aws_glue_catalog.name
44+
database_name = aws_glue_catalog_database.aws_glue_catalog.name
4545
name = "${var.name}-glue-data-catalog-crawler"
4646
role = aws_iam_role.glue_job_role.arn
4747
schedule = "cron(0 12 * * ? *)"
@@ -82,13 +82,13 @@ resource "aws_glue_job" "python_shell_job" {
8282
}
8383

8484
default_arguments = {
85-
"--job-language" = "python" # Default is python
86-
"--additional-python-modules" = replace(file(abspath("${path.module}/../etls/loadFIPS/requirements.txt")), "\n", ", ")
87-
"--MAX_RETRIES" = "3"
88-
"--DB_SECRET_ARN" = module.rds.db_instance_master_user_secret_arn
89-
"--DB_HOST" = module.rds.db_instance_address
90-
"--DB_PORT" = module.rds.db_instance_port
91-
"--DB_NAME" = var.db_name
85+
"--job-language" = "python" # Default is python
86+
"--additional-python-modules" = replace(file(abspath("${path.module}/../etls/loadFIPS/requirements.txt")), "\n", ", ")
87+
"--MAX_RETRIES" = "3"
88+
"--DB_SECRET_ARN" = module.rds.db_instance_master_user_secret_arn
89+
"--DB_HOST" = module.rds.db_instance_address
90+
"--DB_PORT" = module.rds.db_instance_port
91+
"--DB_NAME" = var.db_name
9292
}
9393

9494
execution_property {
@@ -101,15 +101,15 @@ resource "aws_glue_job" "python_shell_job" {
101101
}
102102

103103
resource "aws_glue_job" "pyspark_job" {
104-
name = "nppes-to-s3-pyspark-job"
105-
description = "A simple pyspark job that moves a single table from one location to another"
106-
glue_version = "5.0"
107-
role_arn = aws_iam_role.glue_job_role.arn
104+
name = "nppes-to-s3-pyspark-job"
105+
description = "A simple pyspark job that moves a single table from one location to another"
106+
glue_version = "5.0"
107+
role_arn = aws_iam_role.glue_job_role.arn
108108
number_of_workers = 2
109-
worker_type = "G.1X"
110-
max_retries = 0
111-
timeout = 2880
112-
connections = []
109+
worker_type = "G.1X"
110+
max_retries = 0
111+
timeout = 2880
112+
connections = []
113113

114114
command {
115115
script_location = "s3://${aws_s3_object.glue_job_script.bucket}/${aws_s3_object.glue_job_script_pyspark.key}"
@@ -183,13 +183,13 @@ resource "aws_iam_policy" "glue_job_policy" {
183183
}
184184

185185
resource "aws_iam_policy_attachment" "glue_job_policy_attachment" {
186-
name = "glue_job_policy_attachment"
186+
name = "glue_job_policy_attachment"
187187
policy_arn = aws_iam_policy.glue_job_policy.arn
188-
roles = [aws_iam_role.glue_job_role.name]
188+
roles = [aws_iam_role.glue_job_role.name]
189189
}
190190

191191
resource "aws_iam_policy_attachment" "glue_job_managed_policy_attachment" {
192-
name = "glue_job_managed_policy_attachment"
192+
name = "glue_job_managed_policy_attachment"
193193
policy_arn = "arn:aws-us-gov:iam::aws:policy/service-role/AWSGlueServiceRole"
194-
roles = [aws_iam_role.glue_job_role.name]
194+
roles = [aws_iam_role.glue_job_role.name]
195195
}

0 commit comments

Comments
 (0)