Skip to content

Commit 0167b9e

Browse files
prabinoidprabinoid
authored
Merge pull request #6776 from hotosm/ci/feat/terragrunt-multi-module
Ci/feat/terragrunt multi module
2 parents 6da5d8b + a57fd52 commit 0167b9e

File tree

8 files changed

+378
-73
lines changed

8 files changed

+378
-73
lines changed

‎.github/workflows/terragrunt-apply.yml‎

Lines changed: 60 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,76 @@
11
name: Terragrunt Apply
22

33
on:
4+
workflow_run:
5+
workflows: ["Terragrunt Plan"]
6+
types:
7+
- completed
48
workflow_dispatch:
59
inputs:
610
plan_file_name:
711
description: "Enter Terragrunt Plan File name to run"
812
required: true
913

1014
jobs:
15+
# workaround for using GHActions environment variables feature. In future we can use ${{ github.ref_name }} directly in the workflow as INFRA_BRANCH
16+
get_deployment_meta:
17+
name: Get Deployment Meta
18+
runs-on: ubuntu-latest
19+
outputs:
20+
PLAN_NAME: ${{ steps.export_meta.outputs.PLAN_NAME }}
21+
INFRA_BRANCH_NAME: ${{ steps.export_triggering_wf_meta.outputs.INFRA_BRANCH_NAME }}
22+
INFRA_BRANCH_URL: ${{ steps.export_triggering_wf_meta.outputs.INFRA_BRANCH_URL }}
23+
INFRA_MODULE_PATH: ${{ steps.export_triggering_wf_meta.outputs.INFRA_MODULE_PATH }}
24+
25+
steps:
26+
- name: Export Deployment Meta
27+
id: export_meta
28+
shell: bash
29+
run: |
30+
case "${{ github.event_name }}" in
31+
workflow_run)
32+
export PLAN_NAME=tasking-manager-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
33+
;;
34+
workflow_dispatch)
35+
export PLAN_NAME=${{ inputs.plan_file_name }}
36+
;;
37+
esac
38+
echo "PLAN_NAME=${PLAN_NAME}" >> $GITHUB_OUTPUT
39+
40+
- name: Get Input Params from GitHub Artifacts
41+
uses: actions/download-artifact@v4
42+
with:
43+
name: PLAN_WORKFLOW_META.info
44+
github-token: ${{ github.token }}
45+
repository: ${{ github.repository }}
46+
run-id: ${{ github.event.workflow_run.id }}
47+
48+
- name: Export triggering plan workflow meta
49+
id: export_triggering_wf_meta
50+
shell: bash
51+
run: |
52+
for line in $(cat PLAN_WORKFLOW_META);
53+
do
54+
echo $line >> $GITHUB_OUTPUT
55+
done
56+
1157
apply:
12-
uses: naxa-developers/gh-workflows/.github/workflows/terragrunt-apply.yml@v0.4
58+
name: Terragrunt Apply
59+
uses: hotosm/gh-workflows/.github/workflows/terragrunt-apply.yml@3.1.0
60+
permissions:
61+
id-token: write
62+
contents: read
63+
needs:
64+
- get_deployment_meta
1365
with:
14-
working_dir: ./scripts/aws/infra/staging/purgeable/ecs/
66+
working_dir: ./scripts/aws/infra/${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_NAME }}/${{ needs.get_deployment_meta.outputs.INFRA_MODULE_PATH }}
1567
terraform_version: "1.9.5"
1668
terragrunt_version: "0.67.15"
17-
aws_region: us-west-2
18-
plan_file_name: ${{ inputs.plan_file_name }}
69+
aws_region: us-east-1
70+
load_env: true
71+
plan_file_name: ${{ needs.get_deployment_meta.outputs.PLAN_NAME }}
1972
use_gh_artifacts: true
73+
environment_name: ${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_NAME }}
74+
environment_url: ${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_URL }}
75+
decrypt_plan_file: true
2076
secrets: inherit

‎.github/workflows/terragrunt-plan.yml‎

Lines changed: 62 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,73 @@ name: Terragrunt Plan
22

33
on:
44
workflow_dispatch:
5+
inputs:
6+
module_path:
7+
description: "Select Terragrunt Module to run"
8+
required: true
9+
type: choice
10+
default: purgeable/ecs/
11+
options:
12+
- purgeable/ecs/
13+
- purgeable/ecs-cron/
14+
- non-purgeable/extras/
515

616
jobs:
17+
# workaround for using GHActions environment variables feature. In future we can use ${{ github.ref_name }} directly in the workflow as INFRA_BRANCH
18+
get_deployment_meta:
19+
name: Get Deployment Meta
20+
runs-on: ubuntu-latest
21+
outputs:
22+
INFRA_BRANCH_NAME: ${{ steps.export_meta.outputs.INFRA_BRANCH_NAME }}
23+
INFRA_BRANCH_URL: ${{ steps.export_meta.outputs.INFRA_BRANCH_URL }}
24+
steps:
25+
- name: Export Deployment Meta
26+
id: export_meta
27+
shell: bash
28+
run: |
29+
case "${{ github.ref }}" in
30+
refs/heads/develop)
31+
export INFRA_BRANCH_NAME=staging
32+
export INFRA_BRANCH_URL=https://tasks-stage.hotosm.org
33+
;;
34+
refs/heads/tasking-manager-fastapi)
35+
export INFRA_BRANCH_NAME=staging
36+
export INFRA_BRANCH_URL=https://tasks-stage.hotosm.org
37+
;;
38+
esac
39+
echo "INFRA_BRANCH_NAME=${INFRA_BRANCH_NAME}" >> $GITHUB_OUTPUT
40+
echo "INFRA_BRANCH_URL=${INFRA_BRANCH_URL}" >> $GITHUB_OUTPUT
41+
42+
- name: Write Inputs as Artifacts
43+
shell: bash
44+
run: |
45+
set -e
46+
echo INFRA_BRANCH_NAME=${{ steps.export_meta.outputs.INFRA_BRANCH_NAME }} >> PLAN_WORKFLOW_META
47+
echo INFRA_BRANCH_URL=${{ steps.export_meta.outputs.INFRA_BRANCH_URL }} >> PLAN_WORKFLOW_META
48+
echo INFRA_MODULE_PATH=${{ inputs.module_path }} >> PLAN_WORKFLOW_META
49+
50+
- name: Upload Inputs as Artifacts
51+
uses: actions/upload-artifact@v4
52+
with:
53+
name: PLAN_WORKFLOW_META.info
54+
path: ./PLAN_WORKFLOW_META
55+
retention-days: 1
56+
757
plan:
8-
uses: naxa-developers/gh-workflows/.github/workflows/terragrunt-plan.yml@v0.4
58+
name: Terragrunt Plan
59+
uses: hotosm/gh-workflows/.github/workflows/terragrunt-plan.yml@3.1.0
60+
permissions:
61+
id-token: write
62+
contents: read
63+
needs:
64+
- get_deployment_meta
965
with:
10-
working_dir: ./scripts/aws/infra/staging/purgeable/ecs/
66+
working_dir: ./scripts/aws/infra/${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_NAME }}/${{ github.event.inputs.module_path }}
1167
terraform_version: "1.9.5"
1268
terragrunt_version: "0.67.15"
13-
aws_region: us-west-2
69+
aws_region: us-east-1
1470
load_env: true
71+
environment_name: ${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_NAME }}
72+
environment_url: ${{ needs.get_deployment_meta.outputs.INFRA_BRANCH_URL }}
73+
encrypt_plan_file: true
1574
secrets: inherit

‎scripts/aws/infra/_envcommon/ecs-cron.hcl‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
# ---------------------------------------------------------------------------------------------------------------------
2+
# COMMON TERRAGRUNT CONFIGURATION
3+
# This is the common component configuration for mysql. The common variables for each environment to
4+
# deploy mysql are defined here. This configuration will be merged into the environment configuration
5+
# via an include block.
6+
# ---------------------------------------------------------------------------------------------------------------------
7+
8+
# Terragrunt will copy the Terraform configurations specified by the source parameter, along with any files in the
9+
# working directory, into a temporary folder, and execute your Terraform commands in that folder. If any environment
10+
# needs to deploy a different module version, it should redefine this block with a different ref to override the
11+
# deployed version.
12+
terraform {
13+
source = "${local.base_source_url}?ref=tasking-manager-infra"
14+
}
15+
16+
# ---------------------------------------------------------------------------------------------------------------------
17+
# Locals are named constants that are reusable within the configuration.
18+
# ---------------------------------------------------------------------------------------------------------------------
19+
locals {
20+
# Automatically load environment-level variables
21+
environment_vars = read_terragrunt_config(find_in_parent_folders("deployment_env.hcl"))
22+
23+
# Extract out common variables for reuse
24+
environment = local.environment_vars.locals.environment
25+
application = local.environment_vars.locals.application
26+
team = local.environment_vars.locals.team
27+
28+
# Expose the base source URL so different versions of the module can be deployed in different environments. This will
29+
# be used to construct the terraform block in the child terragrunt configurations.
30+
base_source_url = "git::https://github.com/hotosm/terraform-aws-ecs/"
31+
}
32+
33+
# ---------------------------------------------------------------------------------------------------------------------
34+
# MODULE PARAMETERS
35+
# These are the variables we have to pass in to use the module. This defines the parameters that are common across all
36+
# environments.
37+
# ---------------------------------------------------------------------------------------------------------------------
38+
# Defaults, overridden by env.hcl
39+
40+
inputs = {
41+
service_name = format("%s-%s-%s-%s", local.application, local.team, local.environment, "backend")
42+
43+
log_configuration = {
44+
logdriver = "awslogs"
45+
options = {
46+
awslogs-group = format("%s-%s-%s-%s", local.application, local.team, local.environment, "cron")
47+
awslogs-region = local.environment_vars.locals.aws_region
48+
awslogs-stream-prefix = "cron"
49+
}
50+
}
51+
52+
container_settings = {
53+
app_port = 80
54+
cpu_architecture = "X86_64"
55+
image_url = "ghcr.io/hotosm/tasking-manager-backend"
56+
image_tag = "fastapi"
57+
service_name = format("%s-%s-%s-%s", local.application, local.team, local.environment, "cron")
58+
}
59+
60+
container_capacity = {
61+
cpu = 2048
62+
memory_mb = 4096
63+
}
64+
}

‎scripts/aws/infra/_envcommon/ecs.hcl‎

Lines changed: 0 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -70,19 +70,4 @@ inputs = {
7070
cpu = 2048
7171
memory_mb = 4096
7272
}
73-
74-
## Scaling Policies enabled for cpu,memory in addition to ALB Count.
75-
scale_by_cpu = {
76-
enabled = true
77-
cpu_pct = 70
78-
scale_in_cooldown = 30
79-
scale_out_cooldown = 60
80-
}
81-
82-
scale_by_memory = {
83-
enabled = true
84-
memory_pct = 80
85-
scale_in_cooldown = 30
86-
scale_out_cooldown = 30
87-
}
8873
}

‎scripts/aws/infra/staging/deployment_env.hcl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ locals {
22
account_name = "hotosm"
33
aws_profile = "default"
44
aws_region = "us-east-1"
5-
team = "hotosm"
5+
team = get_env("TEAM", "hotosm")
66
owner = "HOTOSM"
77
environment = "staging"
88
project = "tasking-manager"

‎scripts/aws/infra/staging/purgeable/common-ecs-env.hcl‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
locals {
2+
envs = {
3+
EXTRA_CORS_ORIGINS = get_env("EXTRA_CORS_ORIGINS" ,"[\"https://tasks-stage.hotosm.org\", \"https://tm-ecs-frontend.naxa.com.np\", \"http://localhost:3000\"]")
4+
TM_SMTP_HOST = get_env("TM_SMTP_HOST" ,"email-smtp.us-east-1.amazonaws.com")
5+
TM_SMTP_PORT = get_env("TM_SMTP_PORT" ,"587")
6+
TM_SMTP_USE_TLS = get_env("TM_SMTP_USE_TLS" ,"1")
7+
TM_SMTP_USE_SSL = get_env("TM_SMTP_USE_SSL" ,"0")
8+
TM_EMAIL_FROM_ADDRESS = get_env("TM_EMAIL_FROM_ADDRESS", "noreply@hotosmmail.org")
9+
TM_EMAIL_CONTACT_ADDRESS = get_env("TM_EMAIL_CONTACT_ADDRESS", "sysadmin@hotosm.org")
10+
TM_APP_BASE_URL = get_env("TM_APP_BASE_URL" ,"https://tasks-stage.hotosm.org")
11+
TM_APP_API_URL = get_env("TM_APP_API_URL" ,"https://tasking-manager-staging-api.hotosm.org/api")
12+
TM_REDIRECT_URI = get_env("TM_REDIRECT_URI" ,"https://tasks-stage.hotosm.org/authorized")
13+
TM_APP_API_VERSION = get_env("TM_APP_API_VERSION" ,"v2")
14+
TM_ORG_NAME = get_env("TM_ORG_NAME" ,"Humanitarian OpenStreetMap Team")
15+
TM_ORG_CODE = get_env("TM_ORG_CODE" ,"HOT")
16+
TM_ORG_LOGO = get_env("TM_ORG_LOGO" ,"https://cdn.hotosm.org/tasking-manager/uploads/1588741335578_hot-logo.png")
17+
TM_ORG_URL = get_env("TM_ORG_URL" ,"https://www.hotosm.org/")
18+
TM_ORG_PRIVACY_POLICY_URL = get_env("TM_ORG_PRIVACY_POLICY_URL" ,"https://www.hotosm.org/privacy")
19+
TM_ORG_TWITTER = get_env("TM_ORG_TWITTER" ,"http://twitter.com/hotosm")
20+
TM_ORG_FB = get_env("TM_ORG_FB" ,"https://www.facebook.com/hotosm")
21+
TM_ORG_INSTAGRAM = get_env("TM_ORG_INSTAGRAM" ,"https://www.instagram.com/open.mapping.hubs/")
22+
TM_ORG_YOUTUBE = get_env("TM_ORG_YOUTUBE" ,"https://www.youtube.com/user/hotosm")
23+
TM_ORG_GITHUB = get_env("TM_ORG_GITHUB" ,"https://github.com/hotosm")
24+
OSM_SERVER_URL = get_env("OSM_SERVER_URL" ,"https://www.openstreetmap.org")
25+
OSM_SERVER_API_URL = get_env("OSM_SERVER_API_URL" ,"https://api.openstreetmap.org")
26+
OSM_NOMINATIM_SERVER_URL = get_env("OSM_NOMINATIM_SERVER_URL" ,"https://nominatim.openstreetmap.org")
27+
OSM_REGISTER_URL = get_env("OSM_REGISTER_URL" ,"https://www.openstreetmap.org/user/new")
28+
POSTGRES_TEST_DB = get_env("POSTGRES_TEST_DB" ,"tasking-manager-test")
29+
UNDERPASS_URL = get_env("UNDERPASS_URL" ,"https://underpass.hotosm.org")
30+
TM_SEND_PROJECT_EMAIL_UPDATES = get_env("TM_SEND_PROJECT_EMAIL_UPDATES" ,"1")
31+
TM_DEFAULT_LOCALE = get_env("TM_DEFAULT_LOCALE" ,"en")
32+
TM_LOG_LEVEL = get_env("TM_LOG_LEVEL" , "10")
33+
TM_LOG_DIR = get_env("TM_LOG_DIR", "/var/log/tasking-manager-logs")
34+
TM_SUPPORTED_LANGUAGES_CODES = get_env("TM_SUPPORTED_LANGUAGES_CODES", "en, es")
35+
TM_SUPPORTED_LANGUAGES = get_env("TM_SUPPORTED_LANGUAGES", "English, Español")
36+
TM_DEFAULT_CHANGESET_COMMENT = get_env("TM_DEFAULT_CHANGESET_COMMENT", "#hot-tm-stage-project")
37+
TM_ENVIRONMENT = get_env("TM_ENVIRONMENT", "tasking-manager-staging")
38+
NEW_RELIC_ENVIRONMENT = get_env("TM_ENVIRONMENT", "tasking-manager-staging")
39+
NEW_RELIC_CONFIG_FILE = get_env("NEW_RELIC_CONFIG_FILE", "./scripts/aws/cloudformation/newrelic.ini")
40+
USE_SENTRY = get_env("USE_SENTRY", "false")
41+
# Uncomment the following as needed.
42+
# TM_TASK_AUTOUNLOCK_AFTER = get_env("TM_TASK_AUTOUNLOCK_AFTER", "2h")
43+
# TM_MAPPER_LEVEL_INTERMEDIATE = get_env("TM_MAPPER_LEVEL_INTERMEDIATE", "250")
44+
# TM_MAPPER_LEVEL_ADVANCED = get_env("TM_MAPPER_LEVEL_ADVANCED", "500")
45+
# TM_IMPORT_MAX_FILESIZE = get_env("TM_IMPORT_MAX_FILESIZE", "1000000")
46+
# TM_MAX_AOI_AREA = get_env("TM_MAX_AOI_AREA", "5000")
47+
# EXPORT_TOOL_S3_URL = get_env("EXPORT_TOOL_S3_URL", "https://foorawdataapi.s3.amazonaws.com")
48+
# ENABLE_EXPORT_TOOL = get_env("ENABLE_EXPORT_TOOL", "1")
49+
}
50+
}

‎scripts/aws/infra/staging/purgeable/ecs-cron/terragrunt.hcl‎

Lines changed: 95 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,95 @@
1+
# ---------------------------------------------------------------------------------------------------------------------
2+
# TERRAGRUNT CONFIGURATION
3+
# This is the configuration for Terragrunt, a thin wrapper for Terraform and OpenTofu that helps keep your code DRY and
4+
# maintainable: https://github.com/gruntwork-io/terragrunt
5+
# ---------------------------------------------------------------------------------------------------------------------
6+
7+
# Include the root `terragrunt.hcl` configuration. The root configuration contains settings that are common across all
8+
# components and environments, such as how to configure remote state.
9+
include "root" {
10+
path = find_in_parent_folders("root.hcl")
11+
}
12+
13+
# Include the envcommon configuration for the component. The envcommon configuration contains settings that are common
14+
# for the component across all environments.
15+
include "envcommon" {
16+
path = "${dirname(find_in_parent_folders("root.hcl"))}/_envcommon/ecs-cron.hcl"
17+
# We want to reference the variables from the included config in this configuration, so we expose it.
18+
expose = true
19+
}
20+
21+
# Configure the version of the module to use in this environment. This allows you to promote new versions one
22+
# environment at a time (e.g., qa -> stage -> prod).
23+
terraform {
24+
source = "${include.envcommon.locals.base_source_url}?ref=tasking-manager-infra"
25+
}
26+
27+
locals {
28+
# Automatically load environment-level variables
29+
environment_vars = read_terragrunt_config(find_in_parent_folders("deployment_env.hcl"))
30+
common_ecs_envs = read_terragrunt_config(find_in_parent_folders("common-ecs-env.hcl"))
31+
}
32+
33+
# ---------------------------------------------------------------------------------------------------------------------
34+
# We don't need to override any of the common parameters for this environment, so we don't specify any other parameters.
35+
# ---------------------------------------------------------------------------------------------------------------------
36+
37+
dependency "vpc" {
38+
config_path = "../../non-purgeable/vpc"
39+
}
40+
41+
dependency "alb" {
42+
config_path = "../../non-purgeable/alb"
43+
}
44+
45+
dependency "rds" {
46+
config_path = "../../non-purgeable/rds"
47+
}
48+
49+
dependency "extras" {
50+
config_path = "../../non-purgeable/extras"
51+
}
52+
53+
## Add in any new inputs that you want to overide.
54+
inputs = {
55+
# Inputs from dependencies (Rarely changed)
56+
service_subnets = dependency.vpc.outputs.private_subnets
57+
aws_vpc_id = dependency.vpc.outputs.vpc_id
58+
service_security_groups = [ dependency.alb.outputs.load_balancer_app_security_group ]
59+
deployment_environment = local.environment_vars.locals.environment
60+
61+
task_role_arn = dependency.extras.outputs.ecs_task_role_arn
62+
63+
service_security_groups = [
64+
dependency.alb.outputs.load_balancer_app_security_group
65+
]
66+
67+
# Merge secrets with: key:ValueFrom together
68+
container_secrets = concat(dependency.extras.outputs.container_secrets,
69+
dependency.rds.outputs.database_config_as_ecs_secrets_inputs)
70+
71+
container_commands = [
72+
"sh",
73+
"-c",
74+
"python3 backend/cron_jobs.py"
75+
]
76+
77+
## Task count for ECS services.
78+
tasks_count = {
79+
desired_count = 1
80+
min_healthy_pct = 100
81+
max_pct = 200
82+
}
83+
84+
## Scaling Policy Target Values
85+
scaling_target_values = {
86+
container_min_count = 1
87+
container_max_count = 1
88+
}
89+
90+
# Merge non-sensetive together
91+
container_envvars = merge(
92+
dependency.rds.outputs.database_config_as_ecs_inputs,
93+
local.common_ecs_envs.locals.envs
94+
)
95+
}

0 commit comments

Comments
 (0)